self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-21 09:38:01 +02:00
Code Issues Releases Activity

Desktop, Cli: Fixed importing certain ENEX files that contain invalid dates

Browse Source
This commit is contained in:
Laurent Cozic 2020-11-23 11:23:56 +00:00
parent 4ac2409318
commit 3e65e1539b
7 changed files with 206 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.eslintignore
View File

@ -151,12 +151,9 @@ packages/app-cli/app/LinkSelector.js.map
packages/app-cli/app/services/plugins/PluginRunner.d.ts packages/app-cli/app/services/plugins/PluginRunner.d.ts
packages/app-cli/app/services/plugins/PluginRunner.js packages/app-cli/app/services/plugins/PluginRunner.js
packages/app-cli/app/services/plugins/PluginRunner.js.map packages/app-cli/app/services/plugins/PluginRunner.js.map
packages/app-cli/build/LinkSelector.d.ts packages/app-cli/tests/EnexToMd.d.ts
packages/app-cli/build/LinkSelector.js packages/app-cli/tests/EnexToMd.js
packages/app-cli/build/LinkSelector.js.map packages/app-cli/tests/EnexToMd.js.map
packages/app-cli/build/services/plugins/PluginRunner.d.ts
packages/app-cli/build/services/plugins/PluginRunner.js
packages/app-cli/build/services/plugins/PluginRunner.js.map
packages/app-cli/tests/InMemoryCache.d.ts packages/app-cli/tests/InMemoryCache.d.ts
packages/app-cli/tests/InMemoryCache.js packages/app-cli/tests/InMemoryCache.js
packages/app-cli/tests/InMemoryCache.js.map packages/app-cli/tests/InMemoryCache.js.map

9
.gitignore vendored
View File

@ -143,12 +143,9 @@ packages/app-cli/app/LinkSelector.js.map
packages/app-cli/app/services/plugins/PluginRunner.d.ts packages/app-cli/app/services/plugins/PluginRunner.d.ts
packages/app-cli/app/services/plugins/PluginRunner.js packages/app-cli/app/services/plugins/PluginRunner.js
packages/app-cli/app/services/plugins/PluginRunner.js.map packages/app-cli/app/services/plugins/PluginRunner.js.map
packages/app-cli/build/LinkSelector.d.ts packages/app-cli/tests/EnexToMd.d.ts
packages/app-cli/build/LinkSelector.js packages/app-cli/tests/EnexToMd.js
packages/app-cli/build/LinkSelector.js.map packages/app-cli/tests/EnexToMd.js.map
packages/app-cli/build/services/plugins/PluginRunner.d.ts
packages/app-cli/build/services/plugins/PluginRunner.js
packages/app-cli/build/services/plugins/PluginRunner.js.map
packages/app-cli/tests/InMemoryCache.d.ts packages/app-cli/tests/InMemoryCache.d.ts
packages/app-cli/tests/InMemoryCache.js packages/app-cli/tests/InMemoryCache.js
packages/app-cli/tests/InMemoryCache.js.map packages/app-cli/tests/InMemoryCache.js.map

67
packages/app-cli/tests/EnexToMd.js
View File

@ -1,67 +0,0 @@
/* eslint-disable no-unused-vars */
const os = require('os');
const time = require('@joplin/lib/time').default;
const { filename } = require('@joplin/lib/path-utils');
const { asyncTest, fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('./test-utils.js');
const Folder = require('@joplin/lib/models/Folder.js');
const Note = require('@joplin/lib/models/Note.js');
const BaseModel = require('@joplin/lib/BaseModel').default;
const shim = require('@joplin/lib/shim').default;
const { enexXmlToMd } = require('@joplin/lib/import-enex-md-gen.js');
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
describe('EnexToMd', function() {
beforeEach(async (done) => {
await setupDatabaseAndSynchronizer(1);
await switchClient(1);
done();
});
it('should convert from Enex to Markdown', asyncTest(async () => {
const basePath = `${__dirname}/enex_to_md`;
const files = await shim.fsDriver().readDirStats(basePath);
for (let i = 0; i < files.length; i++) {
const htmlFilename = files[i].path;
if (htmlFilename.indexOf('.html') < 0) continue;
const htmlPath = `${basePath}/${htmlFilename}`;
const mdPath = `${basePath}/${filename(htmlFilename)}.md`;
// if (htmlFilename !== 'multiline_inner_text.html') continue;
const html = await shim.fsDriver().readFile(htmlPath);
let expectedMd = await shim.fsDriver().readFile(mdPath);
let actualMd = await enexXmlToMd(`<div>${html}</div>`, []);
if (os.EOL === '\r\n') {
expectedMd = expectedMd.replace(/\r\n/g, '\n');
actualMd = actualMd.replace(/\r\n/g, '\n');
}
if (actualMd !== expectedMd) {
console.info('');
console.info(`Error converting file: ${htmlFilename}`);
console.info('--------------------------------- Got:');
console.info(actualMd.split('\n'));
console.info('--------------------------------- Expected:');
console.info(expectedMd.split('\n'));
console.info('--------------------------------------------');
console.info('');
expect(false).toBe(true);
// return;
} else {
expect(true).toBe(true);
}
}
}));
});

99
packages/app-cli/tests/EnexToMd.ts Normal file
View File

@ -0,0 +1,99 @@
import { NoteEntity, ResourceEntity, TagEntity } from '@joplin/lib/services/database/types';
import shim from '@joplin/lib/shim';
const fs = require('fs-extra');
const os = require('os');
const { filename } = require('@joplin/lib/path-utils');
const { setupDatabaseAndSynchronizer, switchClient } = require('./test-utils.js');
const { enexXmlToMd } = require('@joplin/lib/import-enex-md-gen.js');
const { importEnex } = require('@joplin/lib/import-enex');
const Note = require('@joplin/lib/models/Note');
const Tag = require('@joplin/lib/models/Tag');
const Resource = require('@joplin/lib/models/Resource');
const enexSampleBaseDir = `${__dirname}/enex_to_md`;
describe('EnexToMd', function() {
beforeEach(async (done) => {
await setupDatabaseAndSynchronizer(1);
await switchClient(1);
done();
});
it('should convert ENEX content to Markdown', async () => {
const files = await shim.fsDriver().readDirStats(enexSampleBaseDir);
for (let i = 0; i < files.length; i++) {
const htmlFilename = files[i].path;
if (htmlFilename.indexOf('.html') < 0) continue;
const htmlPath = `${enexSampleBaseDir}/${htmlFilename}`;
const mdPath = `${enexSampleBaseDir}/${filename(htmlFilename)}.md`;
// if (htmlFilename !== 'multiline_inner_text.html') continue;
const html = await shim.fsDriver().readFile(htmlPath);
let expectedMd = await shim.fsDriver().readFile(mdPath);
let actualMd = await enexXmlToMd(`<div>${html}</div>`, []);
if (os.EOL === '\r\n') {
expectedMd = expectedMd.replace(/\r\n/g, '\n');
actualMd = actualMd.replace(/\r\n/g, '\n');
}
if (actualMd !== expectedMd) {
console.info('');
console.info(`Error converting file: ${htmlFilename}`);
console.info('--------------------------------- Got:');
console.info(actualMd.split('\n'));
console.info('--------------------------------- Expected:');
console.info(expectedMd.split('\n'));
console.info('--------------------------------------------');
console.info('');
expect(false).toBe(true);
// return;
} else {
expect(true).toBe(true);
}
}
});
it('should import ENEX metadata', async () => {
const filePath = `${enexSampleBaseDir}/sample-enex.xml`;
await importEnex('', filePath);
const note: NoteEntity = (await Note.all())[0];
expect(note.title).toBe('Test Note for Export');
expect(note.body).toBe([
' Hello, World.',
'',
'![snapshot-DAE9FC15-88E3-46CF-B744-DA9B1B56EB57.jpg](:/3d0f4d01abc02cf8c4dc1c796df8c4b2)',
].join('\n'));
expect(note.created_time).toBe(1375217524000);
expect(note.updated_time).toBe(1376560800000);
expect(note.latitude).toBe('33.88394692');
expect(note.longitude).toBe('-117.91913551');
expect(note.altitude).toBe('96.0000');
expect(note.author).toBe('Brett Kelly');
const tag: TagEntity = (await Tag.tagsByNoteId(note.id))[0];
expect(tag.title).toBe('fake-tag');
const resource: ResourceEntity = (await Resource.all())[0];
expect(resource.id).toBe('3d0f4d01abc02cf8c4dc1c796df8c4b2');
const stat = await fs.stat(Resource.fullPath(resource));
expect(stat.size).toBe(277);
});
it('should handle invalid dates', async () => {
const filePath = `${enexSampleBaseDir}/invalid_date.enex`;
await importEnex('', filePath);
const note: NoteEntity = (await Note.all())[0];
expect(note.created_time).toBe(1521822724000); // 20180323T163204Z
expect(note.updated_time).toBe(1521822724000); // Because this date was invalid, it is set to the created time instead
});
});

14
packages/app-cli/tests/enex_to_md/invalid_date.enex Normal file
View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export3.dtd">
<en-export export-date="20201223T053642Z" application="Evernote" version="10.4.3">
<note>
<title>Fruit Tree Assessment</title>
<created>20180323T163204Z</created>
<updated>10101T000000Z</updated>
<content>
<![CDATA[<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd"
><en-note>Fruit Tree</en-note> ]]>
</content>
</note>
</en-export>

47
packages/app-cli/tests/enex_to_md/sample-enex.xml Executable file
View File

@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export3.dtd">
<en-export export-date="20130730T205637Z" application="Evernote" version="Evernote Mac">
<note>
<title>Test Note for Export</title>
<content>
<![CDATA[<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
<en-note style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;">
Hello, World.
<div>
<br/>
</div>
<div>
<en-media alt="" type="image/jpeg" hash="dd7b6d285d09ec054e8cd6a3814ce093"/>
</div>
<div>
<br/>
</div>
</en-note>
]]>
</content>
<created>20130730T205204Z</created>
<updated>20130815T100000Z</updated>
<tag>fake-tag</tag>
<note-attributes>
<latitude>33.88394692352314</latitude>
<longitude>-117.9191355110099</longitude>
<altitude>96</altitude>
<author>Brett Kelly</author>
</note-attributes>
<resource>
<data encoding="base64">/9j/4AAQSkZJRgABAQAAAQABAAD/4gxYSUNDX1BST0ZJTEUAAQEAAAxITGlubwIQAABtbnRyUkdCIFhZ
WiAHzgACAAkABgAxAABhY3NwTVNGVAAAAABJRUMgc1JHQgAAAAAAAAAAAAAAAAAA9tYAAQAAAADTLUhQ
kfeIGT/+uufk8DpM0gyVjGfmzkgetesnUoTHJ+5Cxn86zmv4/wB75EW+QHAPUH/P9Ky+s1rtrr/wfvOm
dBSamnq/xPKp/hpLKmS7x4OBjgn6elee6v4OuLJirRSHb/FtyG9s9u1fR0+oTiIRvGq7W4bpisfUGk1C
GVWtkIyM57n1rfDY+uqigtU76ffZkUsA6iajHZ6v/P8A4B//2Q==</data>
<mime>image/jpeg</mime>
<width>1280</width>
<height>720</height>
<resource-attributes>
<file-name>snapshot-DAE9FC15-88E3-46CF-B744-DA9B1B56EB57.jpg</file-name>
</resource-attributes>
</resource>
</note>
</en-export>

60
packages/lib/import-enex.js
View File

@ -18,7 +18,7 @@ const shim = require('./shim').default;
// const Promise = require('promise'); // const Promise = require('promise');
const fs = require('fs-extra'); const fs = require('fs-extra');
function dateToTimestamp(s, zeroIfInvalid = false) { function dateToTimestamp(s, defaultValue = null) {
// Most dates seem to be in this format // Most dates seem to be in this format
let m = moment(s, 'YYYYMMDDTHHmmssZ'); let m = moment(s, 'YYYYMMDDTHHmmssZ');
@ -27,7 +27,7 @@ function dateToTimestamp(s, zeroIfInvalid = false) {
if (!m.isValid()) m = moment(s, 'YYYYMMDDThmmss AZ'); if (!m.isValid()) m = moment(s, 'YYYYMMDDThmmss AZ');
if (!m.isValid()) { if (!m.isValid()) {
if (zeroIfInvalid) return 0; if (defaultValue !== null) return defaultValue;
throw new Error(`Invalid date: ${s}`); throw new Error(`Invalid date: ${s}`);
} }
@ -258,6 +258,20 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
if (!('onProgress' in importOptions)) importOptions.onProgress = function() {}; if (!('onProgress' in importOptions)) importOptions.onProgress = function() {};
if (!('onError' in importOptions)) importOptions.onError = function() {}; if (!('onError' in importOptions)) importOptions.onError = function() {};
const handleSaxStreamEvent = (fn) => {
return function(...args) {
try {
fn(...args);
} catch (error) {
if (importOptions.onError) {
importOptions.onError(error);
} else {
console.error(error);
}
}
};
};
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const progressState = { const progressState = {
loaded: 0, loaded: 0,
@ -337,9 +351,15 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
note.parent_id = parentFolderId; note.parent_id = parentFolderId;
note.body = body; note.body = body;
// Notes in enex files always have a created timestamp but not always an // If the created timestamp was invalid, it would be
// updated timestamp (it the note has never been modified). For sync // set to zero, so set it to the current date here
// we require an updated_time property, so set it to create_time in that case if (!note.created_time) note.created_time = Date.now();
// Notes in enex files always have a created timestamp
// but not always an updated timestamp (it the note has
// never been modified). For sync we require an
// updated_time property, so set it to create_time in
// that case
if (!note.updated_time) note.updated_time = note.created_time; if (!note.updated_time) note.updated_time = note.created_time;
const result = await saveNoteToStorage(note, importOptions); const result = await saveNoteToStorage(note, importOptions);
@ -368,7 +388,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
importOptions.onError(error); importOptions.onError(error);
}); });
saxStream.on('text', function(text) { saxStream.on('text', handleSaxStreamEvent(function(text) {
const n = currentNodeName(); const n = currentNodeName();
if (noteAttributes) { if (noteAttributes) {
@ -395,9 +415,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
if (n == 'title') { if (n == 'title') {
note.title = text; note.title = text;
} else if (n == 'created') { } else if (n == 'created') {
note.created_time = dateToTimestamp(text); note.created_time = dateToTimestamp(text, 0);
} else if (n == 'updated') { } else if (n == 'updated') {
note.updated_time = dateToTimestamp(text); note.updated_time = dateToTimestamp(text, 0);
} else if (n == 'tag') { } else if (n == 'tag') {
note.tags.push(text); note.tags.push(text);
} else if (n == 'note') { } else if (n == 'note') {
@ -408,9 +428,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
console.warn(`Unsupported note tag: ${n}`); console.warn(`Unsupported note tag: ${n}`);
} }
} }
}); }));
saxStream.on('opentag', function(node) { saxStream.on('opentag', handleSaxStreamEvent(function(node) {
const n = node.name.toLowerCase(); const n = node.name.toLowerCase();
nodes.push(node); nodes.push(node);
@ -428,9 +448,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
} else if (n == 'resource') { } else if (n == 'resource') {
noteResource = {}; noteResource = {};
} }
}); }));
saxStream.on('cdata', function(data) { saxStream.on('cdata', handleSaxStreamEvent(function(data) {
const n = currentNodeName(); const n = currentNodeName();
if (noteResourceRecognition) { if (noteResourceRecognition) {
@ -444,9 +464,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
} }
} }
} }
}); }));
saxStream.on('closetag', async function(n) { saxStream.on('closetag', handleSaxStreamEvent(function(n) {
nodes.pop(); nodes.pop();
if (n == 'note') { if (n == 'note') {
@ -476,9 +496,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
note.altitude = noteAttributes.altitude; note.altitude = noteAttributes.altitude;
note.author = noteAttributes.author ? noteAttributes.author.trim() : ''; note.author = noteAttributes.author ? noteAttributes.author.trim() : '';
note.is_todo = noteAttributes['reminder-order'] !== '0' && !!noteAttributes['reminder-order']; note.is_todo = noteAttributes['reminder-order'] !== '0' && !!noteAttributes['reminder-order'];
note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], true); note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], 0);
note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], true); note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], 0);
note.order = dateToTimestamp(noteAttributes['reminder-order'], true); note.order = dateToTimestamp(noteAttributes['reminder-order'], 0);
note.source = noteAttributes.source ? `evernote.${noteAttributes.source.trim()}` : 'evernote'; note.source = noteAttributes.source ? `evernote.${noteAttributes.source.trim()}` : 'evernote';
note.source_url = noteAttributes['source-url'] ? noteAttributes['source-url'].trim() : ''; note.source_url = noteAttributes['source-url'] ? noteAttributes['source-url'].trim() : '';
@ -495,9 +515,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
noteResource = null; noteResource = null;
} }
}); }));
saxStream.on('end', function() { saxStream.on('end', handleSaxStreamEvent(function() {
// Wait till there is no more notes to process. // Wait till there is no more notes to process.
const iid = shim.setInterval(() => { const iid = shim.setInterval(() => {
processNotes().then(allDone => { processNotes().then(allDone => {
@ -507,7 +527,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
} }
}); });
}, 500); }, 500);
}); }));
stream.pipe(saxStream); stream.pipe(saxStream);
}); });