All: Fixes #9151: Import of inter-linked md files has incorrect notebook structure (#9269)

packages/lib/services/interop/InteropService_Importer_Md.test.ts

@@ -10,14 +10,29 @@ import { FolderEntity } from '../database/types';
 describe('InteropService_Importer_Md', () => {
 	let tempDir: string;
 	async function importNote(path: string) {
+		const newFolder = await Folder.save({ title: 'folder' });
 		const importer = new InteropService_Importer_Md();
-		importer.setMetadata({ fileExtensions: ['md', 'html'] });
+		await importer.init(path, {
-		return await importer.importFile(path, 'notebook');
+			format: 'md',
+			outputFormat: 'md',
+			path,
+			destinationFolder: newFolder,
+			destinationFolderId: newFolder.id,
+		});
+		importer.setMetadata({ fileExtensions: ['md'] });
+		await importer.exec({ warnings: [] });
+		const allNotes = await Note.all();
+		return allNotes[0];
 	}
 	async function importNoteDirectory(path: string) {
 		const importer = new InteropService_Importer_Md();
+		await importer.init(path, {
+			format: 'md',
+			outputFormat: 'md',
+			path,
+		});
 		importer.setMetadata({ fileExtensions: ['md', 'html'] });
-		return await importer.importDirectory(path, 'notebook');
+		return await importer.exec({ warnings: [] });
 	}
 	beforeEach(async () => {
 		await setupDatabaseAndSynchronizer(1);
@@ -77,10 +92,8 @@ describe('InteropService_Importer_Md', () => {
 		expect(noteIds.length).toBe(1);
 	});
 	it('should gracefully handle reference cycles in notes', async () => {
-		const importer = new InteropService_Importer_Md();
+		await importNoteDirectory(`${supportDir}/test_notes/md/cycle-reference`);
-		importer.setMetadata({ fileExtensions: ['md'] });
+		const [noteA, noteB] = await Note.all();
-		const noteA = await importer.importFile(`${supportDir}/test_notes/md/sample-cycles-a.md`, 'notebook');
-		const noteB = await importer.importFile(`${supportDir}/test_notes/md/sample-cycles-b.md`, 'notebook');
 
 		const noteAIds = await Note.linkedNoteIds(noteA.body);
 		expect(noteAIds.length).toBe(1);
@@ -137,11 +150,12 @@ describe('InteropService_Importer_Md', () => {
 		expect(allFolders.map((f: FolderEntity) => f.title).indexOf('non-empty')).toBeGreaterThanOrEqual(0);
 	});
 	it('should not import empty directory', async () => {
-		await fs.mkdirp(`${tempDir}/empty/empty`);
+		await fs.mkdirp(`${tempDir}/empty1/empty2`);
 
-		await importNoteDirectory(`${tempDir}/empty`);
+		await importNoteDirectory(`${tempDir}/empty1`);
 		const allFolders = await Folder.all();
-		expect(allFolders.map((f: FolderEntity) => f.title).indexOf('empty')).toBe(-1);
+		expect(allFolders.map((f: FolderEntity) => f.title).indexOf('empty1')).toBe(0);
+		expect(allFolders.map((f: FolderEntity) => f.title).indexOf('empty2')).toBe(-1);
 	});
 	it('should import directory with non-empty subdirectory', async () => {
 		await fs.mkdirp(`${tempDir}/non-empty-subdir/non-empty-subdir/subdir-empty`);
@@ -154,4 +168,20 @@ describe('InteropService_Importer_Md', () => {
 		expect(allFolders.map((f: FolderEntity) => f.title).indexOf('subdir-empty')).toBe(-1);
 		expect(allFolders.map((f: FolderEntity) => f.title).indexOf('subdir-non-empty')).toBeGreaterThanOrEqual(0);
 	});
+
+	it('should import all files before replacing links', async () => {
+		await fs.mkdirp(`${tempDir}/links/0/1/2`);
+		await fs.mkdirp(`${tempDir}/links/Target_folder`);
+		await fs.writeFile(`${tempDir}/links/Target_folder/Targeted_note.md`, '# Targeted_note');
+		await fs.writeFile(`${tempDir}/links/0/1/2/Note_with_reference_to_another_note.md`, '# 20\n[Target_folder:Targeted_note](../../../Target_folder/Targeted_note.md)');
+
+		await importNoteDirectory(`${tempDir}/links`);
+
+		const allFolders = await Folder.all();
+		const allNotes = await Note.all();
+		const targetFolder = allFolders.find(f => f.title === 'Target_folder');
+		const noteBeingReferenced = allNotes.find(n => n.title === 'Targeted_note');
+
+		expect(noteBeingReferenced.parent_id).toBe(targetFolder.id);
+	});
 });

packages/lib/services/interop/InteropService_Importer_Md.ts

@@ -14,7 +14,7 @@ const { pregQuote } = require('../../string-utils-common');
 import { MarkupToHtml } from '@joplin/renderer';
 
 export default class InteropService_Importer_Md extends InteropService_Importer_Base {
-	private importedNotes: Record<string, NoteEntity> = {};
+	protected importedNotes: Record<string, NoteEntity> = {};
 
 	public async exec(result: ImportExportResult) {
 		let parentFolderId = null;
@@ -42,6 +42,16 @@ export default class InteropService_Importer_Md extends InteropService_Importer_
 			await this.importFile(filePaths[i], parentFolderId);
 		}
 
+		for (const importedLocalPath of Object.keys(this.importedNotes)) {
+			const note = this.importedNotes[importedLocalPath];
+			const updatedBody = await this.importLocalFiles(importedLocalPath, note.body, note.parent_id);
+			const updatedNote = {
+				...this.importedNotes[importedLocalPath],
+				body: updatedBody || note.body,
+			};
+			this.importedNotes[importedLocalPath] = await Note.save(updatedNote, { isNew: false, autoTimestamp: false });
+		}
+
 		return result;
 	}
 
@@ -97,7 +107,7 @@ export default class InteropService_Importer_Md extends InteropService_Importer_
 		const markdownLinks = markdownUtils.extractFileUrls(md);
 		const htmlLinks = htmlUtils.extractFileUrls(md);
 		const fileLinks = unique(markdownLinks.concat(htmlLinks));
-		await Promise.all(fileLinks.map(async (encodedLink: string) => {
+		for (const encodedLink of fileLinks) {
 			const link = decodeURI(encodedLink);
 			// Handle anchor links appropriately
 			const trimmedLink = this.trimAnchorLink(link);
@@ -138,7 +148,7 @@ export default class InteropService_Importer_Md extends InteropService_Importer_
 					updated = htmlUtils.replaceResourceUrl(updated, linkToReplace, id);
 				}
 			}
-		}));
+		}
 		return updated;
 	}
 
@@ -163,17 +173,6 @@ export default class InteropService_Importer_Md extends InteropService_Importer_
 		};
 		this.importedNotes[resolvedPath] = await Note.save(note, { autoTimestamp: false });
 
-		try {
-			const updatedBody = await this.importLocalFiles(resolvedPath, body, parentFolderId);
-			const updatedNote = {
-				...this.importedNotes[resolvedPath],
-				body: updatedBody || body,
-			};
-			this.importedNotes[resolvedPath] = await Note.save(updatedNote, { isNew: false });
-		} catch (error) {
-			// console.error(`Problem importing links for file ${resolvedPath}, error:\n ${error}`);
-		}
-
 		return this.importedNotes[resolvedPath];
 	}
 }

packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts

@@ -1,13 +1,24 @@
-import InteropService_Importer_Md_frontmatter from '../../services/interop/InteropService_Importer_Md_frontmatter';
 import Note from '../../models/Note';
 import Tag from '../../models/Tag';
 import time from '../../time';
 import { setupDatabaseAndSynchronizer, supportDir, switchClient } from '../../testing/test-utils';
+import { ImportModuleOutputFormat, ImportOptions } from './types';
+import InteropService from './InteropService';
+import Folder from '../../models/Folder';
 
 async function importNote(path: string) {
-	const importer = new InteropService_Importer_Md_frontmatter();
+	const folder = await Folder.save({});
-	importer.setMetadata({ fileExtensions: ['md', 'html'] });
+	const importOptions: ImportOptions = {
-	return await importer.importFile(path, 'notebook');
+		path: path,
+		format: 'md_frontmatter',
+		destinationFolderId: folder.id,
+		outputFormat: ImportModuleOutputFormat.Markdown,
+	};
+
+	await InteropService.instance().import(importOptions);
+
+	const allNotes = await Note.all();
+	return allNotes[0];
 }
 
 const importTestFile = async (name: string) => {
@@ -32,7 +43,7 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
 		expect(note.longitude).toBe('-94.51350100');
 		expect(note.altitude).toBe('0.0000');
 		expect(note.is_todo).toBe(1);
-		expect(note.todo_completed).toBeUndefined();
+		expect(note.todo_completed).toBe(0);
 		expect(time.formatMsToLocal(note.todo_due, format)).toBe('22/08/2021 00:00');
 		expect(note.body).toBe('This is the note body\n');
 
@@ -84,7 +95,7 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => {
 
 		expect(note.longitude).toBe('-94.51350100');
 		expect(note.is_todo).toBe(1);
-		expect(note.todo_completed).toBeUndefined();
+		expect(note.todo_completed).toBe(0);
 	});
 	it('should load notes with newline in the title', async () => {
 		const note = await importTestFile('title_newline.md');

packages/lib/services/interop/InteropService_Importer_Md_frontmatter.ts

@@ -5,6 +5,7 @@ import time from '../../time';
 import { NoteEntity } from '../database/types';
 
 import * as yaml from 'js-yaml';
+import shim from '../../shim';
 
 interface ParsedMeta {
 	metadata: NoteEntity;
@@ -162,6 +163,9 @@ export default class InteropService_Importer_Md_frontmatter extends InteropServi
 
 		const noteItem = await Note.save(updatedNote, { isNew: false, autoTimestamp: false });
 
+		const resolvedPath = shim.fsDriver().resolve(filePath);
+		this.importedNotes[resolvedPath] = noteItem;
+
 		for (const tag of tags) { await Tag.addNoteTagByTitle(noteItem.id, tag); }
 
 		return noteItem;