Desktop, Cli: Fixes #8802: Improved import of invalid Markdown+FrontMatter files

packages/app-cli/tests/support/test_notes/yaml/multiple_newlines_after_marker.md

@@ -0,0 +1,7 @@
+---
+title: Example
+---
+
+
+
+note body

packages/app-cli/tests/support/test_notes/yaml/no_newline_after_marker.md

@@ -0,0 +1,4 @@
+---
+title: Example
+---
+note body

packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts

@@ -4,20 +4,23 @@ import Tag from '../../models/Tag';
 import time from '../../time';
 import { setupDatabaseAndSynchronizer, supportDir, switchClient } from '../../testing/test-utils';
 
+async function importNote(path: string) {
+	const importer = new InteropService_Importer_Md_frontmatter();
+	importer.setMetadata({ fileExtensions: ['md', 'html'] });
+	return await importer.importFile(path, 'notebook');
+}
+
+const importTestFile = async (name: string) => {
+	return importNote(`${supportDir}/test_notes/yaml/${name}`);
+};
 
 describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
-	async function importNote(path: string) {
-		const importer = new InteropService_Importer_Md_frontmatter();
-		importer.setMetadata({ fileExtensions: ['md', 'html'] });
-		return await importer.importFile(path, 'notebook');
-	}
-
 	beforeEach(async () => {
 		await setupDatabaseAndSynchronizer(1);
 		await switchClient(1);
 	});
 	it('should import file and set all metadata correctly', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/full.md`);
+		const note = await importTestFile('full.md');
 		const format = 'DD/MM/YYYY HH:mm';
 
 		expect(note.title).toBe('Test Note Title');
@@ -42,14 +45,14 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
 		expect(tagTitles).toContain('pencil');
 	});
 	it('should only import data from the first yaml block', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/split.md`);
+		const note = await importTestFile('split.md');
 
 		expect(note.title).toBe('xxx');
 		expect(note.author).not.toBe('xxx');
 		expect(note.body).toBe('---\nauthor: xxx\n---\n\nnote body\n');
 	});
 	it('should only import, duplicate notes and tags are not created', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/duplicates.md`);
+		const note = await importTestFile('duplicates.md');
 
 		expect(note.title).toBe('ddd');
 		const itemIds = await Note.linkedItemIds(note.body);
@@ -59,13 +62,13 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
 		expect(tags.length).toBe(1);
 	});
 	it('should not import items as numbers', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/numbers.md`);
+		const note = await importTestFile('numbers.md');
 
 		expect(note.title).toBe('001');
 		expect(note.body).toBe('note body\n');
 	});
 	it('should normalize whitespace and load correctly', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/normalize.md`);
+		const note = await importTestFile('normalize.md');
 
 		expect(note.title).toBe('norm');
 		expect(note.body).toBe('note body\n');
@@ -74,7 +77,7 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
 		expect(tags.length).toBe(3);
 	});
 	it('should load unquoted special forms correctly', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/unquoted.md`);
+		const note = await importTestFile('unquoted.md');
 
 		expect(note.title).toBe('Unquoted');
 		expect(note.body).toBe('note body\n');
@@ -84,19 +87,19 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
 		expect(note.todo_completed).toBeUndefined();
 	});
 	it('should load notes with newline in the title', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/title_newline.md`);
+		const note = await importTestFile('title_newline.md');
 
 		expect(note.title).toBe('First\nSecond');
 	});
 	it('should import dates (without time) correctly', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/short_date.md`);
+		const note = await importTestFile('short_date.md');
 		const format = 'YYYY-MM-DD HH:mm';
 
 		expect(time.formatMsToLocal(note.user_updated_time, format)).toBe('2021-01-01 00:00');
 		expect(time.formatMsToLocal(note.user_created_time, format)).toBe('2017-01-01 00:00');
 	});
 	it('should load tags even with the inline syntax', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/inline_tags.md`);
+		const note = await importTestFile('inline_tags.md');
 
 		expect(note.title).toBe('Inline Tags');
 
@@ -104,7 +107,7 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
 		expect(tags.length).toBe(2);
 	});
 	it('should import r-markdown files correctly and set what metadata it can', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/r-markdown.md`);
+		const note = await importTestFile('r-markdown.md');
 		const format = 'YYYY-MM-DD HH:mm';
 
 		expect(note.title).toBe('YAML metadata for R Markdown with examples');
@@ -120,15 +123,25 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
 		expect(tagTitles).toContain('rmd');
 	});
 	it('should import r-markdown files with alternative author syntax', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/r-markdown_author.md`);
+		const note = await importTestFile('r-markdown_author.md');
 
 		expect(note.title).toBe('Distill for R Markdown');
 		expect(note.author).toBe('JJ Allaire');
 	});
 	it('should handle date formats with timezone information', async () => {
-		const note = await importNote(`${supportDir}/test_notes/yaml/utc.md`);
+		const note = await importTestFile('utc.md');
 
 		expect(note.user_updated_time).toBe(1556729640000);
 		expect(note.user_created_time).toBe(1556754840000);
 	});
+
+	it('should accept file with no newline after the block marker', async () => {
+		const note = await importTestFile('no_newline_after_marker.md');
+		expect(note.body).toBe('note body\n');
+	});
+
+	it('should handle multiple newlines before the note body', async () => {
+		const note = await importTestFile('multiple_newlines_after_marker.md');
+		expect(note.body).toBe('\n\nnote body');
+	});
 });

packages/lib/services/interop/InteropService_Importer_Md_frontmatter.ts

@@ -59,9 +59,16 @@ export default class InteropService_Importer_Md_frontmatter extends InteropServi
 		const bodyLines: string[] = [];
 		for (let i = 0; i < lines.length; i++) {
 			const line = lines[i];
+			const nextLine = i + 1 <= lines.length - 1 ? lines[i + 1] : '';
+
 			if (inHeader && line.startsWith('---')) {
 				inHeader = false;
-				i++; // Need to eat the extra newline after the yaml block
+
+				// Need to eat the extra newline after the yaml block. Note that
+				// if the next line is not an empty line, we keep it. Fixes
+				// https://github.com/laurent22/joplin/issues/8802
+				if (nextLine.trim() === '') i++;
+
 				continue;
 			}
 

readme/spec/interop_with_frontmatter.md

@@ -12,8 +12,12 @@ tags:
   - export
   - import
 ---
+
+Note body
 ```
 
+There should be an empty line between the `---` delimiter and the note body. Any empty line after that will be considered to be part of the note body. When importing notes, if there is no empty lines between the `---` delimiter and the note body, everything directly after `---` will be considered to be the note body.
+
 ## Supported Metadata Fields
 
 All of the below fields are supported by both the exporter and the importer.