From eceb14ff9ec4939952dd67b33a925d95b5bfe2c7 Mon Sep 17 00:00:00 2001 From: JackGruber <24863925+JackGruber@users.noreply.github.com> Date: Wed, 19 May 2021 23:22:03 +0200 Subject: [PATCH] Desktop: Resolves #4433: Import linked local files when importing Markdown files (#4966) --- .eslintignore | 6 ++++ .gitignore | 6 ++++ .../app-cli/tests/{MdToMd.js => MdToMd.ts} | 5 +++ .../{markdownUtils.js => markdownUtils.ts} | 36 ++++++++++++++++++- .../app-cli/tests/md_to_md/sample-files.md | 9 +++++ packages/lib/markdownUtils.ts | 11 +++--- .../interop/InteropService_Importer_Md.ts | 8 ++--- 7 files changed, 72 insertions(+), 9 deletions(-) rename packages/app-cli/tests/{MdToMd.js => MdToMd.ts} (90%) rename packages/app-cli/tests/{markdownUtils.js => markdownUtils.ts} (68%) create mode 100644 packages/app-cli/tests/md_to_md/sample-files.md diff --git a/.eslintignore b/.eslintignore index 166a18fa0..743fea51a 100644 --- a/.eslintignore +++ b/.eslintignore @@ -86,6 +86,9 @@ packages/app-cli/tests/InMemoryCache.js.map packages/app-cli/tests/MdToHtml.d.ts packages/app-cli/tests/MdToHtml.js packages/app-cli/tests/MdToHtml.js.map +packages/app-cli/tests/MdToMd.d.ts +packages/app-cli/tests/MdToMd.js +packages/app-cli/tests/MdToMd.js.map packages/app-cli/tests/Synchronizer.basics.d.ts packages/app-cli/tests/Synchronizer.basics.js packages/app-cli/tests/Synchronizer.basics.js.map @@ -122,6 +125,9 @@ packages/app-cli/tests/fsDriver.js.map packages/app-cli/tests/htmlUtils.d.ts packages/app-cli/tests/htmlUtils.js packages/app-cli/tests/htmlUtils.js.map +packages/app-cli/tests/markdownUtils.d.ts +packages/app-cli/tests/markdownUtils.js +packages/app-cli/tests/markdownUtils.js.map packages/app-cli/tests/models_Folder.d.ts packages/app-cli/tests/models_Folder.js packages/app-cli/tests/models_Folder.js.map diff --git a/.gitignore b/.gitignore index 62e1ced6d..9201f242b 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,9 @@ packages/app-cli/tests/InMemoryCache.js.map packages/app-cli/tests/MdToHtml.d.ts packages/app-cli/tests/MdToHtml.js packages/app-cli/tests/MdToHtml.js.map +packages/app-cli/tests/MdToMd.d.ts +packages/app-cli/tests/MdToMd.js +packages/app-cli/tests/MdToMd.js.map packages/app-cli/tests/Synchronizer.basics.d.ts packages/app-cli/tests/Synchronizer.basics.js packages/app-cli/tests/Synchronizer.basics.js.map @@ -108,6 +111,9 @@ packages/app-cli/tests/fsDriver.js.map packages/app-cli/tests/htmlUtils.d.ts packages/app-cli/tests/htmlUtils.js packages/app-cli/tests/htmlUtils.js.map +packages/app-cli/tests/markdownUtils.d.ts +packages/app-cli/tests/markdownUtils.js +packages/app-cli/tests/markdownUtils.js.map packages/app-cli/tests/models_Folder.d.ts packages/app-cli/tests/models_Folder.js packages/app-cli/tests/models_Folder.js.map diff --git a/packages/app-cli/tests/MdToMd.js b/packages/app-cli/tests/MdToMd.ts similarity index 90% rename from packages/app-cli/tests/MdToMd.js rename to packages/app-cli/tests/MdToMd.ts index 4873d3e32..e9ee67018 100644 --- a/packages/app-cli/tests/MdToMd.js +++ b/packages/app-cli/tests/MdToMd.ts @@ -43,4 +43,9 @@ describe('InteropService_Importer_Md: importLocalImages', function() { const items = await Note.linkedItems(note.body); expect(items.length).toBe(1); }); + it('should import resources for files', async function() { + const note = await importer.importFile(`${__dirname}/md_to_md/sample-files.md`, 'notebook'); + const items = await Note.linkedItems(note.body); + expect(items.length).toBe(4); + }); }); diff --git a/packages/app-cli/tests/markdownUtils.js b/packages/app-cli/tests/markdownUtils.ts similarity index 68% rename from packages/app-cli/tests/markdownUtils.js rename to packages/app-cli/tests/markdownUtils.ts index 8b6b9624f..cf6cba066 100644 --- a/packages/app-cli/tests/markdownUtils.js +++ b/packages/app-cli/tests/markdownUtils.ts @@ -33,14 +33,48 @@ describe('markdownUtils', function() { ['![something](http://test.com/img.png "Some description")', ['http://test.com/img.png']], ['![something](https://test.com/ohoh_(123).png)', ['https://test.com/ohoh_(123).png']], ['![nothing]() ![something](http://test.com/img.png)', ['http://test.com/img.png']], + ['![something](img.png)', ['img.png']], + ['![something](/img.png)', ['/img.png']], + ['![something](../img.png)', ['../img.png']], + ['![something](../upload/img.png)', ['../upload/img.png']], + ['![something](./upload/img.png)', ['./upload/img.png']], + ['[something](testing.html)', ['']], + ['[something](img.png)', ['']], + ['![something](file://img.png)', ['file://img.png']], ]; for (let i = 0; i < testCases.length; i++) { const md = testCases[i][0]; const actual = markdownUtils.extractImageUrls(md); const expected = testCases[i][1]; + expect(actual.join(' ')).toBe((expected as string[]).join(' ')); + } + })); - expect(actual.join(' ')).toBe(expected.join(' ')); + it('should extract files URLs', (async () => { + const testCases = [ + ['[something](http://test.com/img.png)', ['http://test.com/img.png']], + ['[something](http://test.com/test.txt)', ['http://test.com/test.txt']], + ['[something](http://test.com/img.png) ![something2](http://test.com/img2.png)', ['http://test.com/img.png', 'http://test.com/img2.png']], + ['[something](http://test.com/img.png "Some description")', ['http://test.com/img.png']], + ['[something](https://test.com/ohoh_(123).png)', ['https://test.com/ohoh_(123).png']], + ['[nothing]() ![something](http://test.com/img.png)', ['http://test.com/img.png']], + ['[something](test.txt)', ['test.txt']], + ['[something](/test.txt)', ['/test.txt']], + ['[something](../test.txt)', ['../test.txt']], + ['[something](../upload/test.txt)', ['../upload/test.txt']], + ['[something](./upload/test.txt)', ['./upload/test.txt']], + ['[something](testing.html)', ['testing.html']], + ['[something](img.png)', ['img.png']], + ['[something](file://img.png)', ['file://img.png']], + ]; + + for (let i = 0; i < testCases.length; i++) { + const md = testCases[i][0]; + const actual = markdownUtils.extractFileUrls(md); + const expected = testCases[i][1]; + + expect(actual.join(' ')).toBe((expected as string[]).join(' ')); } })); diff --git a/packages/app-cli/tests/md_to_md/sample-files.md b/packages/app-cli/tests/md_to_md/sample-files.md new file mode 100644 index 000000000..06454a841 --- /dev/null +++ b/packages/app-cli/tests/md_to_md/sample-files.md @@ -0,0 +1,9 @@ +# Markdown file test + +![../support/photo.jpg](../support/photo.jpg) + +[welcome.pdf](../support/welcome.pdf) + +[sample.md](sample.md) + +[sample2.md](./sample.md) diff --git a/packages/lib/markdownUtils.ts b/packages/lib/markdownUtils.ts index 491a54866..a7df5b538 100644 --- a/packages/lib/markdownUtils.ts +++ b/packages/lib/markdownUtils.ts @@ -61,7 +61,7 @@ const markdownUtils = { }, // Returns the **encoded** URLs, so to be useful they should be decoded again before use. - extractImageUrls(md: string) { + extractFileUrls(md: string, onlyImage: boolean = false): Array { const markdownIt = new MarkdownIt(); markdownIt.validateLink = validateLinks; // Necessary to support file:/// links @@ -72,11 +72,10 @@ const markdownUtils = { const searchUrls = (tokens: any[]) => { for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; - - if (token.type === 'image') { + if ((onlyImage === true && token.type === 'image') || (onlyImage === false && (token.type === 'image' || token.type === 'link_open'))) { for (let j = 0; j < token.attrs.length; j++) { const a = token.attrs[j]; - if (a[0] === 'src' && a.length >= 2 && a[1]) { + if ((a[0] === 'src' || a[0] === 'href') && a.length >= 2 && a[1]) { output.push(a[1]); } } @@ -93,6 +92,10 @@ const markdownUtils = { return output; }, + extractImageUrls(md: string) { + return markdownUtils.extractFileUrls(md,true); + }, + // The match results has 5 items // Full match array is // [Full match, whitespace, list token, ol line number, whitespace following token] diff --git a/packages/lib/services/interop/InteropService_Importer_Md.ts b/packages/lib/services/interop/InteropService_Importer_Md.ts index 4356a91dd..e572f19d3 100644 --- a/packages/lib/services/interop/InteropService_Importer_Md.ts +++ b/packages/lib/services/interop/InteropService_Importer_Md.ts @@ -63,10 +63,10 @@ export default class InteropService_Importer_Md extends InteropService_Importer_ * Parse text for links, attempt to find local file, if found create Joplin resource * and update link accordingly. */ - async importLocalImages(filePath: string, md: string) { + async importLocalFiles(filePath: string, md: string) { let updated = md; - const imageLinks = unique(markdownUtils.extractImageUrls(md)); - await Promise.all(imageLinks.map(async (encodedLink: string) => { + const fileLinks = unique(markdownUtils.extractFileUrls(md)); + await Promise.all(fileLinks.map(async (encodedLink: string) => { const link = decodeURI(encodedLink); const attachmentPath = filename(`${dirname(filePath)}/${link}`, true); const pathWithExtension = `${attachmentPath}.${fileExtension(link)}`; @@ -90,7 +90,7 @@ export default class InteropService_Importer_Md extends InteropService_Importer_ const body = await shim.fsDriver().readFile(filePath); let updatedBody; try { - updatedBody = await this.importLocalImages(filePath, body); + updatedBody = await this.importLocalFiles(filePath, body); } catch (error) { // console.error(`Problem importing links for file ${filePath}, error:\n ${error}`); }