From 5ab6a89046eeafe759706abe5ee1f2c07527e24a Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Mon, 25 Sep 2023 13:47:49 +0100 Subject: [PATCH] Desktop, Cli: Fixes #8802: Improved import of invalid Markdown+FrontMatter files --- .../yaml/multiple_newlines_after_marker.md | 7 +++ .../yaml/no_newline_after_marker.md | 4 ++ ...ropService_Importer_Md_frontmatter.test.ts | 49 ++++++++++++------- .../InteropService_Importer_Md_frontmatter.ts | 9 +++- readme/spec/interop_with_frontmatter.md | 4 ++ 5 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 packages/app-cli/tests/support/test_notes/yaml/multiple_newlines_after_marker.md create mode 100644 packages/app-cli/tests/support/test_notes/yaml/no_newline_after_marker.md diff --git a/packages/app-cli/tests/support/test_notes/yaml/multiple_newlines_after_marker.md b/packages/app-cli/tests/support/test_notes/yaml/multiple_newlines_after_marker.md new file mode 100644 index 0000000000..a1db6f3b1a --- /dev/null +++ b/packages/app-cli/tests/support/test_notes/yaml/multiple_newlines_after_marker.md @@ -0,0 +1,7 @@ +--- +title: Example +--- + + + +note body \ No newline at end of file diff --git a/packages/app-cli/tests/support/test_notes/yaml/no_newline_after_marker.md b/packages/app-cli/tests/support/test_notes/yaml/no_newline_after_marker.md new file mode 100644 index 0000000000..e95ab4d7e3 --- /dev/null +++ b/packages/app-cli/tests/support/test_notes/yaml/no_newline_after_marker.md @@ -0,0 +1,4 @@ +--- +title: Example +--- +note body diff --git a/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts b/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts index abe69cc07a..447afbceda 100644 --- a/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts +++ b/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts @@ -4,20 +4,23 @@ import Tag from '../../models/Tag'; import time from '../../time'; import { setupDatabaseAndSynchronizer, supportDir, switchClient } from '../../testing/test-utils'; +async function importNote(path: string) { + const importer = new InteropService_Importer_Md_frontmatter(); + importer.setMetadata({ fileExtensions: ['md', 'html'] }); + return await importer.importFile(path, 'notebook'); +} + +const importTestFile = async (name: string) => { + return importNote(`${supportDir}/test_notes/yaml/${name}`); +}; describe('InteropService_Importer_Md_frontmatter: importMetadata', () => { - async function importNote(path: string) { - const importer = new InteropService_Importer_Md_frontmatter(); - importer.setMetadata({ fileExtensions: ['md', 'html'] }); - return await importer.importFile(path, 'notebook'); - } - beforeEach(async () => { await setupDatabaseAndSynchronizer(1); await switchClient(1); }); it('should import file and set all metadata correctly', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/full.md`); + const note = await importTestFile('full.md'); const format = 'DD/MM/YYYY HH:mm'; expect(note.title).toBe('Test Note Title'); @@ -42,14 +45,14 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => { expect(tagTitles).toContain('pencil'); }); it('should only import data from the first yaml block', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/split.md`); + const note = await importTestFile('split.md'); expect(note.title).toBe('xxx'); expect(note.author).not.toBe('xxx'); expect(note.body).toBe('---\nauthor: xxx\n---\n\nnote body\n'); }); it('should only import, duplicate notes and tags are not created', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/duplicates.md`); + const note = await importTestFile('duplicates.md'); expect(note.title).toBe('ddd'); const itemIds = await Note.linkedItemIds(note.body); @@ -59,13 +62,13 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => { expect(tags.length).toBe(1); }); it('should not import items as numbers', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/numbers.md`); + const note = await importTestFile('numbers.md'); expect(note.title).toBe('001'); expect(note.body).toBe('note body\n'); }); it('should normalize whitespace and load correctly', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/normalize.md`); + const note = await importTestFile('normalize.md'); expect(note.title).toBe('norm'); expect(note.body).toBe('note body\n'); @@ -74,7 +77,7 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => { expect(tags.length).toBe(3); }); it('should load unquoted special forms correctly', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/unquoted.md`); + const note = await importTestFile('unquoted.md'); expect(note.title).toBe('Unquoted'); expect(note.body).toBe('note body\n'); @@ -84,19 +87,19 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => { expect(note.todo_completed).toBeUndefined(); }); it('should load notes with newline in the title', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/title_newline.md`); + const note = await importTestFile('title_newline.md'); expect(note.title).toBe('First\nSecond'); }); it('should import dates (without time) correctly', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/short_date.md`); + const note = await importTestFile('short_date.md'); const format = 'YYYY-MM-DD HH:mm'; expect(time.formatMsToLocal(note.user_updated_time, format)).toBe('2021-01-01 00:00'); expect(time.formatMsToLocal(note.user_created_time, format)).toBe('2017-01-01 00:00'); }); it('should load tags even with the inline syntax', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/inline_tags.md`); + const note = await importTestFile('inline_tags.md'); expect(note.title).toBe('Inline Tags'); @@ -104,7 +107,7 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => { expect(tags.length).toBe(2); }); it('should import r-markdown files correctly and set what metadata it can', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/r-markdown.md`); + const note = await importTestFile('r-markdown.md'); const format = 'YYYY-MM-DD HH:mm'; expect(note.title).toBe('YAML metadata for R Markdown with examples'); @@ -120,15 +123,25 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => { expect(tagTitles).toContain('rmd'); }); it('should import r-markdown files with alternative author syntax', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/r-markdown_author.md`); + const note = await importTestFile('r-markdown_author.md'); expect(note.title).toBe('Distill for R Markdown'); expect(note.author).toBe('JJ Allaire'); }); it('should handle date formats with timezone information', async () => { - const note = await importNote(`${supportDir}/test_notes/yaml/utc.md`); + const note = await importTestFile('utc.md'); expect(note.user_updated_time).toBe(1556729640000); expect(note.user_created_time).toBe(1556754840000); }); + + it('should accept file with no newline after the block marker', async () => { + const note = await importTestFile('no_newline_after_marker.md'); + expect(note.body).toBe('note body\n'); + }); + + it('should handle multiple newlines before the note body', async () => { + const note = await importTestFile('multiple_newlines_after_marker.md'); + expect(note.body).toBe('\n\nnote body'); + }); }); diff --git a/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.ts b/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.ts index 52681710f7..4c5b6b8058 100644 --- a/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.ts +++ b/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.ts @@ -59,9 +59,16 @@ export default class InteropService_Importer_Md_frontmatter extends InteropServi const bodyLines: string[] = []; for (let i = 0; i < lines.length; i++) { const line = lines[i]; + const nextLine = i + 1 <= lines.length - 1 ? lines[i + 1] : ''; + if (inHeader && line.startsWith('---')) { inHeader = false; - i++; // Need to eat the extra newline after the yaml block + + // Need to eat the extra newline after the yaml block. Note that + // if the next line is not an empty line, we keep it. Fixes + // https://github.com/laurent22/joplin/issues/8802 + if (nextLine.trim() === '') i++; + continue; } diff --git a/readme/spec/interop_with_frontmatter.md b/readme/spec/interop_with_frontmatter.md index b5393ca86a..b1ba581315 100644 --- a/readme/spec/interop_with_frontmatter.md +++ b/readme/spec/interop_with_frontmatter.md @@ -12,8 +12,12 @@ tags: - export - import --- + +Note body ``` +There should be an empty line between the `---` delimiter and the note body. Any empty line after that will be considered to be part of the note body. When importing notes, if there is no empty lines between the `---` delimiter and the note body, everything directly after `---` will be considered to be the note body. + ## Supported Metadata Fields All of the below fields are supported by both the exporter and the importer.