1
0
mirror of https://github.com/laurent22/joplin.git synced 2024-11-24 08:12:24 +02:00

Desktop, Cli: Replace note links with relative paths in MD Exporter (#2161)

* Replace linked Note ids by relative paths in MD Exporter.

* Added tests for the MD Exporter.

* Changed fs.readdirSync use for earlier Node version (v8)

In the previous commit the code used fs.readdirSync from Node v10 or
later. But since Joplin still uses v8, I changed the use of
fs.readdirSync to be in line with the earlier api.

* Updated readDirSync use for Node v10, which allows gets folder names too.

* Revert "Updated readDirSync use for Node v10, which allows gets folder names too."

This reverts commit 8f255db120861dd7773d99e1b63f4864d39594cf.
Because the Travis builds still use Node v8. This is fine as well, the
readdirSync returns the filenames in the directory.

* Added reservedNames param to findUniqueFilename
This commit is contained in:
Vaidotas Simkus 2020-01-18 14:16:14 +01:00 committed by Laurent Cozic
parent 69f9e38730
commit d9d75d6c71
6 changed files with 428 additions and 21 deletions

View File

@ -0,0 +1,336 @@
/* eslint-disable no-unused-vars */
require('app-module-path').addPath(__dirname);
const fs = require('fs-extra');
const { asyncTest, setupDatabaseAndSynchronizer, switchClient } = require('test-utils.js');
const InteropService_Exporter_Md = require('lib/services/InteropService_Exporter_Md.js');
const BaseModel = require('lib/BaseModel.js');
const Folder = require('lib/models/Folder.js');
const Resource = require('lib/models/Resource.js');
const Note = require('lib/models/Note.js');
const { shim } = require('lib/shim.js');
jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000;
const exportDir = `${__dirname}/export`;
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
describe('services_InteropService_Exporter_Md', function() {
beforeEach(async (done) => {
await setupDatabaseAndSynchronizer(1);
await switchClient(1);
await fs.remove(exportDir);
await fs.mkdirp(exportDir);
done();
});
it('should create resources directory', asyncTest(async () => {
const service = new InteropService_Exporter_Md();
await service.init(exportDir);
expect(await shim.fsDriver().exists(`${exportDir}/_resources/`)).toBe(true);
}));
it('should create note paths and add them to context', asyncTest(async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir);
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
let folder1 = await Folder.save({ title: 'folder1' });
let note1 = await Note.save({ title: 'note1', parent_id: folder1.id });
let note2 = await Note.save({ title: 'note2', parent_id: folder1.id });
await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`);
note1 = await Note.load(note1.id);
queueExportItem(BaseModel.TYPE_FOLDER, folder1.id);
queueExportItem(BaseModel.TYPE_NOTE, note1);
queueExportItem(BaseModel.TYPE_NOTE, note2);
queueExportItem(BaseModel.TYPE_RESOURCE, (await Note.linkedResourceIds(note1.body))[0]);
let folder2 = await Folder.save({ title: 'folder2' });
let note3 = await Note.save({ title: 'note3', parent_id: folder2.id });
await shim.attachFileToNote(note3, `${__dirname}/../tests/support/photo.jpg`);
note3 = await Note.load(note3.id);
queueExportItem(BaseModel.TYPE_FOLDER, folder2.id);
queueExportItem(BaseModel.TYPE_NOTE, note3);
queueExportItem(BaseModel.TYPE_RESOURCE, (await Note.linkedResourceIds(note3.body))[0]);
expect(!exporter.context() && !(exporter.context().notePaths || Object.keys(exporter.context().notePaths).length)).toBe(false, 'Context should be empty before processing.');
await exporter.processItem(Folder, folder1);
await exporter.processItem(Folder, folder2);
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
expect(Object.keys(exporter.context().notePaths).length).toBe(3, 'There should be 3 note paths in the context.');
expect(exporter.context().notePaths[note1.id]).toBe('folder1/note1.md');
expect(exporter.context().notePaths[note2.id]).toBe('folder1/note2.md');
expect(exporter.context().notePaths[note3.id]).toBe('folder2/note3.md');
}));
it('should handle duplicate note names', asyncTest(async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir);
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
let folder1 = await Folder.save({ title: 'folder1' });
let note1 = await Note.save({ title: 'note1', parent_id: folder1.id });
let note1_2 = await Note.save({ title: 'note1', parent_id: folder1.id });
queueExportItem(BaseModel.TYPE_FOLDER, folder1.id);
queueExportItem(BaseModel.TYPE_NOTE, note1);
queueExportItem(BaseModel.TYPE_NOTE, note1_2);
await exporter.processItem(Folder, folder1);
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
expect(Object.keys(exporter.context().notePaths).length).toBe(2, 'There should be 2 note paths in the context.');
expect(exporter.context().notePaths[note1.id]).toBe('folder1/note1.md');
expect(exporter.context().notePaths[note1_2.id]).toBe('folder1/note1 (1).md');
}));
it('should not override existing files', asyncTest(async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir);
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
let folder1 = await Folder.save({ title: 'folder1' });
let note1 = await Note.save({ title: 'note1', parent_id: folder1.id });
queueExportItem(BaseModel.TYPE_FOLDER, folder1.id);
queueExportItem(BaseModel.TYPE_NOTE, note1);
await exporter.processItem(Folder, folder1);
// Create a file with the path of note1 before processing note1
await shim.fsDriver().writeFile(`${exportDir}/folder1/note1.md`, 'Note content', 'utf-8');
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
expect(Object.keys(exporter.context().notePaths).length).toBe(1, 'There should be 1 note paths in the context.');
expect(exporter.context().notePaths[note1.id]).toBe('folder1/note1 (1).md');
}));
it('should save resource files in _resource directory', asyncTest(async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir);
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
let folder1 = await Folder.save({ title: 'folder1' });
let note1 = await Note.save({ title: 'note1', parent_id: folder1.id });
await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`);
note1 = await Note.load(note1.id);
queueExportItem(BaseModel.TYPE_FOLDER, folder1.id);
queueExportItem(BaseModel.TYPE_NOTE, note1);
queueExportItem(BaseModel.TYPE_RESOURCE, (await Note.linkedResourceIds(note1.body))[0]);
let resource1 = await Resource.load(itemsToExport[2].itemOrId);
let folder2 = await Folder.save({ title: 'folder2', parent_id: folder1.id });
let note2 = await Note.save({ title: 'note2', parent_id: folder2.id });
await shim.attachFileToNote(note2, `${__dirname}/../tests/support/photo.jpg`);
note2 = await Note.load(note2.id);
queueExportItem(BaseModel.TYPE_FOLDER, folder2.id);
queueExportItem(BaseModel.TYPE_NOTE, note2);
queueExportItem(BaseModel.TYPE_RESOURCE, (await Note.linkedResourceIds(note2.body))[0]);
let resource2 = await Resource.load(itemsToExport[5].itemOrId);
await exporter.processResource(resource1, Resource.fullPath(resource1));
await exporter.processResource(resource2, Resource.fullPath(resource2));
expect(await shim.fsDriver().exists(`${exportDir}/_resources/${Resource.filename(resource1)}`)).toBe(true, 'Resource file should be copied to _resources directory.');
expect(await shim.fsDriver().exists(`${exportDir}/_resources/${Resource.filename(resource2)}`)).toBe(true, 'Resource file should be copied to _resources directory.');
}));
it('should save notes in fs', asyncTest(async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir);
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
let folder1 = await Folder.save({ title: 'folder1' });
let note1 = await Note.save({ title: 'note1', parent_id: folder1.id });
queueExportItem(BaseModel.TYPE_FOLDER, folder1.id);
queueExportItem(BaseModel.TYPE_NOTE, note1);
let folder2 = await Folder.save({ title: 'folder2', parent_id: folder1.id });
let note2 = await Note.save({ title: 'note2', parent_id: folder2.id });
queueExportItem(BaseModel.TYPE_FOLDER, folder2.id);
queueExportItem(BaseModel.TYPE_NOTE, note2);
let folder3 = await Folder.save({ title: 'folder3' });
let note3 = await Note.save({ title: 'note3', parent_id: folder3.id });
queueExportItem(BaseModel.TYPE_FOLDER, folder3.id);
queueExportItem(BaseModel.TYPE_NOTE, note3);
await exporter.processItem(Folder, folder1);
await exporter.processItem(Folder, folder2);
await exporter.processItem(Folder, folder3);
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
await exporter.processItem(Note, note1);
await exporter.processItem(Note, note2);
await exporter.processItem(Note, note3);
expect(await shim.fsDriver().exists(`${exportDir}/${exporter.context().notePaths[note1.id]}`)).toBe(true, 'File should be saved in filesystem.');
expect(await shim.fsDriver().exists(`${exportDir}/${exporter.context().notePaths[note2.id]}`)).toBe(true, 'File should be saved in filesystem.');
expect(await shim.fsDriver().exists(`${exportDir}/${exporter.context().notePaths[note3.id]}`)).toBe(true, 'File should be saved in filesystem.');
}));
it('should replace resource ids with relative paths', asyncTest(async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir);
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
let folder1 = await Folder.save({ title: 'folder1' });
let note1 = await Note.save({ title: 'note1', parent_id: folder1.id });
await shim.attachFileToNote(note1, `${__dirname}/../tests/support/photo.jpg`);
note1 = await Note.load(note1.id);
queueExportItem(BaseModel.TYPE_NOTE, note1);
let resource1 = await Resource.load((await Note.linkedResourceIds(note1.body))[0]);
let folder2 = await Folder.save({ title: 'folder2', parent_id: folder1.id });
let note2 = await Note.save({ title: 'note2', parent_id: folder2.id });
await shim.attachFileToNote(note2, `${__dirname}/../tests/support/photo.jpg`);
note2 = await Note.load(note2.id);
queueExportItem(BaseModel.TYPE_NOTE, note2);
let resource2 = await Resource.load((await Note.linkedResourceIds(note2.body))[0]);
await exporter.processItem(Folder, folder1);
await exporter.processItem(Folder, folder2);
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
let context = {
resourcePaths: {},
};
context.resourcePaths[resource1.id] = 'resource1.jpg';
context.resourcePaths[resource2.id] = 'resource2.jpg';
exporter.updateContext(context);
await exporter.processItem(Note, note1);
await exporter.processItem(Note, note2);
let note1_body = await shim.fsDriver().readFile(`${exportDir}/${exporter.context().notePaths[note1.id]}`);
let note2_body = await shim.fsDriver().readFile(`${exportDir}/${exporter.context().notePaths[note2.id]}`);
expect(note1_body).toContain('](../_resources/resource1.jpg)', 'Resource id should be replaced with a relative path.');
expect(note2_body).toContain('](../../_resources/resource2.jpg)', 'Resource id should be replaced with a relative path.');
}));
it('should replace note ids with relative paths', asyncTest(async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir);
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
const changeNoteBodyAndReload = async (note, newBody) => {
note.body = newBody;
await Note.save(note);
return await Note.load(note.id);
};
let folder1 = await Folder.save({ title: 'folder1' });
let note1 = await Note.save({ title: 'note1', parent_id: folder1.id });
let folder2 = await Folder.save({ title: 'folder2', parent_id: folder1.id });
let note2 = await Note.save({ title: 'note2', parent_id: folder2.id });
let folder3 = await Folder.save({ title: 'folder3' });
let note3 = await Note.save({ title: 'note3', parent_id: folder3.id });
note1 = await changeNoteBodyAndReload(note1, `# Some text \n\n [A link to note3](:/${note3.id})`);
note2 = await changeNoteBodyAndReload(note2, `# Some text \n\n [A link to note3](:/${note3.id}) some more text \n ## And some headers \n and [A link to note1](:/${note1.id}) more links`);
note3 = await changeNoteBodyAndReload(note3, `[A link to note3](:/${note2.id})`);
queueExportItem(BaseModel.TYPE_NOTE, note1);
queueExportItem(BaseModel.TYPE_NOTE, note2);
queueExportItem(BaseModel.TYPE_NOTE, note3);
await exporter.processItem(Folder, folder1);
await exporter.processItem(Folder, folder2);
await exporter.processItem(Folder, folder3);
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
await exporter.processItem(Note, note1);
await exporter.processItem(Note, note2);
await exporter.processItem(Note, note3);
let note1_body = await shim.fsDriver().readFile(`${exportDir}/${exporter.context().notePaths[note1.id]}`);
let note2_body = await shim.fsDriver().readFile(`${exportDir}/${exporter.context().notePaths[note2.id]}`);
let note3_body = await shim.fsDriver().readFile(`${exportDir}/${exporter.context().notePaths[note3.id]}`);
expect(note1_body).toContain('](../folder3/note3.md)', 'Note id should be replaced with a relative path.');
expect(note2_body).toContain('](../../folder3/note3.md)', 'Resource id should be replaced with a relative path.');
expect(note2_body).toContain('](../../folder1/note1.md)', 'Resource id should be replaced with a relative path.');
expect(note3_body).toContain('](../folder1/folder2/note2.md)', 'Resource id should be replaced with a relative path.');
}));
it('should url encode relative note links', asyncTest(async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir);
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
let folder1 = await Folder.save({ title: 'folder with space1' });
let note1 = await Note.save({ title: 'note1 name with space', parent_id: folder1.id });
let note2 = await Note.save({ title: 'note2', parent_id: folder1.id, body: `[link](:/${note1.id})` });
queueExportItem(BaseModel.TYPE_NOTE, note1);
queueExportItem(BaseModel.TYPE_NOTE, note2);
await exporter.processItem(Folder, folder1);
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
await exporter.processItem(Note, note1);
await exporter.processItem(Note, note2);
let note2_body = await shim.fsDriver().readFile(`${exportDir}/${exporter.context().notePaths[note2.id]}`);
expect(note2_body).toContain('[link](../folder%20with%20space1/note1%20name%20with%20space.md)', 'Whitespace in URL should be encoded');
}));
});

View File

@ -21,7 +21,10 @@ class FsDriverBase {
return output;
}
async findUniqueFilename(name) {
async findUniqueFilename(name, reservedNames = null) {
if (reservedNames === null) {
reservedNames = [];
}
let counter = 1;
let nameNoExt = filename(name, true);
@ -29,7 +32,8 @@ class FsDriverBase {
if (extension) extension = `.${extension}`;
let nameToTry = nameNoExt + extension;
while (true) {
const exists = await this.exists(nameToTry);
// Check if the filename does not exist in the filesystem and is not reserved
const exists = await this.exists(nameToTry) || reservedNames.includes(nameToTry);
if (!exists) return nameToTry;
nameToTry = `${nameNoExt} (${counter})${extension}`;
counter++;

View File

@ -143,6 +143,10 @@ class Note extends BaseItem {
return this.linkedItemIdsByType(BaseModel.TYPE_RESOURCE, body);
}
static async linkedNoteIds(body) {
return this.linkedItemIdsByType(BaseModel.TYPE_NOTE, body);
}
static async replaceResourceInternalToExternalLinks(body) {
const resourceIds = await this.linkedResourceIds(body);
const Resource = this.getClass('Resource');

View File

@ -341,6 +341,8 @@ class InteropService {
for (let typeOrderIndex = 0; typeOrderIndex < typeOrder.length; typeOrderIndex++) {
const type = typeOrder[typeOrderIndex];
await exporter.prepareForProcessingItemType(type, itemsToExport);
for (let i = 0; i < itemsToExport.length; i++) {
const itemType = itemsToExport[i].type;

View File

@ -3,7 +3,12 @@
const Setting = require('lib/models/Setting');
class InteropService_Exporter_Base {
constructor() {
this.context_ = {};
}
async init(destDir) {}
async prepareForProcessingItemType(type, itemsToExport) {}
async processItem(ItemClass, item) {}
async processResource(resource, filePath) {}
async close() {}
@ -17,7 +22,7 @@ class InteropService_Exporter_Base {
}
updateContext(context) {
this.context_ = context;
this.context_ = Object.assign(this.context_, context);
}
context() {

View File

@ -1,5 +1,5 @@
const InteropService_Exporter_Base = require('lib/services/InteropService_Exporter_Base');
const { basename, friendlySafeFilename, rtrimSlashes } = require('lib/path-utils.js');
const { basename, friendlySafeFilename } = require('lib/path-utils.js');
const BaseModel = require('lib/BaseModel');
const Folder = require('lib/models/Folder');
const Note = require('lib/models/Note');
@ -31,36 +31,92 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base {
}
}
async replaceResourceIdsByRelativePaths_(item) {
const linkedResourceIds = await Note.linkedResourceIds(item.body);
const relativePath = rtrimSlashes(await this.makeDirPath_(item, '..'));
async relaceLinkedItemIdsByRelativePaths_(item) {
const relativePathToRoot = await this.makeDirPath_(item, '..');
let newBody = await this.replaceResourceIdsByRelativePaths_(item.body, relativePathToRoot);
return await this.replaceNoteIdsByRelativePaths_(newBody, relativePathToRoot);
}
async replaceResourceIdsByRelativePaths_(noteBody, relativePathToRoot) {
const linkedResourceIds = await Note.linkedResourceIds(noteBody);
const resourcePaths = this.context() && this.context().resourcePaths ? this.context().resourcePaths : {};
let newBody = item.body;
let createRelativePath = function(resourcePath) {
return `${relativePathToRoot}_resources/${basename(resourcePath)}`;
};
return await this.replaceItemIdsByRelativePaths_(noteBody, linkedResourceIds, resourcePaths, createRelativePath);
}
for (let i = 0; i < linkedResourceIds.length; i++) {
const id = linkedResourceIds[i];
const resourcePath = `${relativePath}/_resources/${basename(resourcePaths[id])}`;
newBody = newBody.replace(new RegExp(`:/${id}`, 'g'), resourcePath);
async replaceNoteIdsByRelativePaths_(noteBody, relativePathToRoot) {
const linkedNoteIds = await Note.linkedNoteIds(noteBody);
const notePaths = this.context() && this.context().notePaths ? this.context().notePaths : {};
let createRelativePath = function(notePath) {
return encodeURI(`${relativePathToRoot}${notePath}`.trim());
};
return await this.replaceItemIdsByRelativePaths_(noteBody, linkedNoteIds, notePaths, createRelativePath);
}
async replaceItemIdsByRelativePaths_(noteBody, linkedItemIds, paths, fn_createRelativePath) {
let newBody = noteBody;
for (let i = 0; i < linkedItemIds.length; i++) {
const id = linkedItemIds[i];
let itemPath = fn_createRelativePath(paths[id]);
newBody = newBody.replace(new RegExp(`:/${id}`, 'g'), itemPath);
}
return newBody;
}
async prepareForProcessingItemType(type, itemsToExport) {
if (type === BaseModel.TYPE_NOTE) {
// Create unique file path for the note
const context = {
notePaths: {},
};
for (let i = 0; i < itemsToExport.length; i++) {
const itemType = itemsToExport[i].type;
if (itemType !== type) continue;
const itemOrId = itemsToExport[i].itemOrId;
const note = typeof itemOrId === 'object' ? itemOrId : await Note.load(itemOrId);
if (!note) continue;
let notePath = `${await this.makeDirPath_(note)}${friendlySafeFilename(note.title, null, true)}.md`;
notePath = await shim.fsDriver().findUniqueFilename(`${this.destDir_}/${notePath}`, Object.values(context.notePaths));
context.notePaths[note.id] = notePath;
}
// Strip the absolute path to export dir and keep only the relative paths
const destDir = this.destDir_;
Object.keys(context.notePaths).map(function(id) {
context.notePaths[id] = context.notePaths[id].substr(destDir.length + 1);
});
this.updateContext(context);
}
}
async processItem(ItemClass, item) {
if ([BaseModel.TYPE_NOTE, BaseModel.TYPE_FOLDER].indexOf(item.type_) < 0) return;
const dirPath = `${this.destDir_}/${await this.makeDirPath_(item)}`;
if (item.type_ === BaseModel.TYPE_FOLDER) {
const dirPath = `${this.destDir_}/${await this.makeDirPath_(item)}`;
if (this.createdDirs_.indexOf(dirPath) < 0) {
await shim.fsDriver().mkdir(dirPath);
this.createdDirs_.push(dirPath);
}
if (this.createdDirs_.indexOf(dirPath) < 0) {
await shim.fsDriver().mkdir(dirPath);
this.createdDirs_.push(dirPath);
}
if (item.type_ === BaseModel.TYPE_NOTE) {
let noteFilePath = `${dirPath}/${friendlySafeFilename(item.title, null, true)}.md`;
noteFilePath = await shim.fsDriver().findUniqueFilename(noteFilePath);
const noteBody = await this.replaceResourceIdsByRelativePaths_(item);
} else if (item.type_ === BaseModel.TYPE_NOTE) {
const notePaths = this.context() && this.context().notePaths ? this.context().notePaths : {};
let noteFilePath = `${this.destDir_}/${notePaths[item.id]}`;
let noteBody = await this.relaceLinkedItemIdsByRelativePaths_(item);
const modNote = Object.assign({}, item, { body: noteBody });
const noteContent = await Note.serializeForEdit(modNote);
await shim.fsDriver().writeFile(noteFilePath, noteContent, 'utf-8');