From 3e65e1539b8a6b002ecaf69912d3884cf1ed657f Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Mon, 23 Nov 2020 11:23:56 +0000 Subject: [PATCH] Desktop, Cli: Fixed importing certain ENEX files that contain invalid dates --- .eslintignore | 9 +- .gitignore | 9 +- packages/app-cli/tests/EnexToMd.js | 67 ------------- packages/app-cli/tests/EnexToMd.ts | 99 +++++++++++++++++++ .../tests/enex_to_md/invalid_date.enex | 14 +++ .../app-cli/tests/enex_to_md/sample-enex.xml | 47 +++++++++ packages/lib/import-enex.js | 60 +++++++---- 7 files changed, 206 insertions(+), 99 deletions(-) delete mode 100644 packages/app-cli/tests/EnexToMd.js create mode 100644 packages/app-cli/tests/EnexToMd.ts create mode 100644 packages/app-cli/tests/enex_to_md/invalid_date.enex create mode 100755 packages/app-cli/tests/enex_to_md/sample-enex.xml diff --git a/.eslintignore b/.eslintignore index a1e0a20179..1ed2b2d0da 100644 --- a/.eslintignore +++ b/.eslintignore @@ -151,12 +151,9 @@ packages/app-cli/app/LinkSelector.js.map packages/app-cli/app/services/plugins/PluginRunner.d.ts packages/app-cli/app/services/plugins/PluginRunner.js packages/app-cli/app/services/plugins/PluginRunner.js.map -packages/app-cli/build/LinkSelector.d.ts -packages/app-cli/build/LinkSelector.js -packages/app-cli/build/LinkSelector.js.map -packages/app-cli/build/services/plugins/PluginRunner.d.ts -packages/app-cli/build/services/plugins/PluginRunner.js -packages/app-cli/build/services/plugins/PluginRunner.js.map +packages/app-cli/tests/EnexToMd.d.ts +packages/app-cli/tests/EnexToMd.js +packages/app-cli/tests/EnexToMd.js.map packages/app-cli/tests/InMemoryCache.d.ts packages/app-cli/tests/InMemoryCache.js packages/app-cli/tests/InMemoryCache.js.map diff --git a/.gitignore b/.gitignore index b35899c532..53f828173e 100644 --- a/.gitignore +++ b/.gitignore @@ -143,12 +143,9 @@ packages/app-cli/app/LinkSelector.js.map packages/app-cli/app/services/plugins/PluginRunner.d.ts packages/app-cli/app/services/plugins/PluginRunner.js packages/app-cli/app/services/plugins/PluginRunner.js.map -packages/app-cli/build/LinkSelector.d.ts -packages/app-cli/build/LinkSelector.js -packages/app-cli/build/LinkSelector.js.map -packages/app-cli/build/services/plugins/PluginRunner.d.ts -packages/app-cli/build/services/plugins/PluginRunner.js -packages/app-cli/build/services/plugins/PluginRunner.js.map +packages/app-cli/tests/EnexToMd.d.ts +packages/app-cli/tests/EnexToMd.js +packages/app-cli/tests/EnexToMd.js.map packages/app-cli/tests/InMemoryCache.d.ts packages/app-cli/tests/InMemoryCache.js packages/app-cli/tests/InMemoryCache.js.map diff --git a/packages/app-cli/tests/EnexToMd.js b/packages/app-cli/tests/EnexToMd.js deleted file mode 100644 index 7297eb7daa..0000000000 --- a/packages/app-cli/tests/EnexToMd.js +++ /dev/null @@ -1,67 +0,0 @@ -/* eslint-disable no-unused-vars */ - - -const os = require('os'); -const time = require('@joplin/lib/time').default; -const { filename } = require('@joplin/lib/path-utils'); -const { asyncTest, fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('./test-utils.js'); -const Folder = require('@joplin/lib/models/Folder.js'); -const Note = require('@joplin/lib/models/Note.js'); -const BaseModel = require('@joplin/lib/BaseModel').default; -const shim = require('@joplin/lib/shim').default; -const { enexXmlToMd } = require('@joplin/lib/import-enex-md-gen.js'); - -process.on('unhandledRejection', (reason, p) => { - console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); -}); - -describe('EnexToMd', function() { - - beforeEach(async (done) => { - await setupDatabaseAndSynchronizer(1); - await switchClient(1); - done(); - }); - - it('should convert from Enex to Markdown', asyncTest(async () => { - const basePath = `${__dirname}/enex_to_md`; - const files = await shim.fsDriver().readDirStats(basePath); - - for (let i = 0; i < files.length; i++) { - const htmlFilename = files[i].path; - if (htmlFilename.indexOf('.html') < 0) continue; - - const htmlPath = `${basePath}/${htmlFilename}`; - const mdPath = `${basePath}/${filename(htmlFilename)}.md`; - - // if (htmlFilename !== 'multiline_inner_text.html') continue; - - const html = await shim.fsDriver().readFile(htmlPath); - let expectedMd = await shim.fsDriver().readFile(mdPath); - - let actualMd = await enexXmlToMd(`
${html}
`, []); - - if (os.EOL === '\r\n') { - expectedMd = expectedMd.replace(/\r\n/g, '\n'); - actualMd = actualMd.replace(/\r\n/g, '\n'); - } - - if (actualMd !== expectedMd) { - console.info(''); - console.info(`Error converting file: ${htmlFilename}`); - console.info('--------------------------------- Got:'); - console.info(actualMd.split('\n')); - console.info('--------------------------------- Expected:'); - console.info(expectedMd.split('\n')); - console.info('--------------------------------------------'); - console.info(''); - - expect(false).toBe(true); - // return; - } else { - expect(true).toBe(true); - } - } - })); - -}); diff --git a/packages/app-cli/tests/EnexToMd.ts b/packages/app-cli/tests/EnexToMd.ts new file mode 100644 index 0000000000..2811c7eeb5 --- /dev/null +++ b/packages/app-cli/tests/EnexToMd.ts @@ -0,0 +1,99 @@ +import { NoteEntity, ResourceEntity, TagEntity } from '@joplin/lib/services/database/types'; +import shim from '@joplin/lib/shim'; + +const fs = require('fs-extra'); +const os = require('os'); +const { filename } = require('@joplin/lib/path-utils'); +const { setupDatabaseAndSynchronizer, switchClient } = require('./test-utils.js'); +const { enexXmlToMd } = require('@joplin/lib/import-enex-md-gen.js'); +const { importEnex } = require('@joplin/lib/import-enex'); +const Note = require('@joplin/lib/models/Note'); +const Tag = require('@joplin/lib/models/Tag'); +const Resource = require('@joplin/lib/models/Resource'); + +const enexSampleBaseDir = `${__dirname}/enex_to_md`; + +describe('EnexToMd', function() { + + beforeEach(async (done) => { + await setupDatabaseAndSynchronizer(1); + await switchClient(1); + done(); + }); + + it('should convert ENEX content to Markdown', async () => { + const files = await shim.fsDriver().readDirStats(enexSampleBaseDir); + + for (let i = 0; i < files.length; i++) { + const htmlFilename = files[i].path; + if (htmlFilename.indexOf('.html') < 0) continue; + + const htmlPath = `${enexSampleBaseDir}/${htmlFilename}`; + const mdPath = `${enexSampleBaseDir}/${filename(htmlFilename)}.md`; + + // if (htmlFilename !== 'multiline_inner_text.html') continue; + + const html = await shim.fsDriver().readFile(htmlPath); + let expectedMd = await shim.fsDriver().readFile(mdPath); + + let actualMd = await enexXmlToMd(`
${html}
`, []); + + if (os.EOL === '\r\n') { + expectedMd = expectedMd.replace(/\r\n/g, '\n'); + actualMd = actualMd.replace(/\r\n/g, '\n'); + } + + if (actualMd !== expectedMd) { + console.info(''); + console.info(`Error converting file: ${htmlFilename}`); + console.info('--------------------------------- Got:'); + console.info(actualMd.split('\n')); + console.info('--------------------------------- Expected:'); + console.info(expectedMd.split('\n')); + console.info('--------------------------------------------'); + console.info(''); + + expect(false).toBe(true); + // return; + } else { + expect(true).toBe(true); + } + } + }); + + it('should import ENEX metadata', async () => { + const filePath = `${enexSampleBaseDir}/sample-enex.xml`; + await importEnex('', filePath); + + const note: NoteEntity = (await Note.all())[0]; + expect(note.title).toBe('Test Note for Export'); + expect(note.body).toBe([ + ' Hello, World.', + '', + '![snapshot-DAE9FC15-88E3-46CF-B744-DA9B1B56EB57.jpg](:/3d0f4d01abc02cf8c4dc1c796df8c4b2)', + ].join('\n')); + expect(note.created_time).toBe(1375217524000); + expect(note.updated_time).toBe(1376560800000); + expect(note.latitude).toBe('33.88394692'); + expect(note.longitude).toBe('-117.91913551'); + expect(note.altitude).toBe('96.0000'); + expect(note.author).toBe('Brett Kelly'); + + const tag: TagEntity = (await Tag.tagsByNoteId(note.id))[0]; + expect(tag.title).toBe('fake-tag'); + + const resource: ResourceEntity = (await Resource.all())[0]; + expect(resource.id).toBe('3d0f4d01abc02cf8c4dc1c796df8c4b2'); + const stat = await fs.stat(Resource.fullPath(resource)); + expect(stat.size).toBe(277); + }); + + it('should handle invalid dates', async () => { + const filePath = `${enexSampleBaseDir}/invalid_date.enex`; + await importEnex('', filePath); + const note: NoteEntity = (await Note.all())[0]; + expect(note.created_time).toBe(1521822724000); // 20180323T163204Z + expect(note.updated_time).toBe(1521822724000); // Because this date was invalid, it is set to the created time instead + }); + +}); diff --git a/packages/app-cli/tests/enex_to_md/invalid_date.enex b/packages/app-cli/tests/enex_to_md/invalid_date.enex new file mode 100644 index 0000000000..fb83911816 --- /dev/null +++ b/packages/app-cli/tests/enex_to_md/invalid_date.enex @@ -0,0 +1,14 @@ + + + + + Fruit Tree Assessment + 20180323T163204Z + 10101T000000Z + + +Fruit Tree ]]> + + + diff --git a/packages/app-cli/tests/enex_to_md/sample-enex.xml b/packages/app-cli/tests/enex_to_md/sample-enex.xml new file mode 100755 index 0000000000..14081850a9 --- /dev/null +++ b/packages/app-cli/tests/enex_to_md/sample-enex.xml @@ -0,0 +1,47 @@ + + + + + Test Note for Export + + + + + Hello, World. +
+
+
+
+ +
+
+
+
+ + ]]> + + 20130730T205204Z + 20130815T100000Z + fake-tag + + 33.88394692352314 + -117.9191355110099 + 96 + Brett Kelly + + + /9j/4AAQSkZJRgABAQAAAQABAAD/4gxYSUNDX1BST0ZJTEUAAQEAAAxITGlubwIQAABtbnRyUkdCIFhZ + WiAHzgACAAkABgAxAABhY3NwTVNGVAAAAABJRUMgc1JHQgAAAAAAAAAAAAAAAAAA9tYAAQAAAADTLUhQ + kfeIGT/+uufk8DpM0gyVjGfmzkgetesnUoTHJ+5Cxn86zmv4/wB75EW+QHAPUH/P9Ky+s1rtrr/wfvOm + dBSamnq/xPKp/hpLKmS7x4OBjgn6elee6v4OuLJirRSHb/FtyG9s9u1fR0+oTiIRvGq7W4bpisfUGk1C + GVWtkIyM57n1rfDY+uqigtU76ffZkUsA6iajHZ6v/P8A4B//2Q== + image/jpeg + 1280 + 720 + + snapshot-DAE9FC15-88E3-46CF-B744-DA9B1B56EB57.jpg + + + + + diff --git a/packages/lib/import-enex.js b/packages/lib/import-enex.js index 2c84ada48d..2813bfe048 100644 --- a/packages/lib/import-enex.js +++ b/packages/lib/import-enex.js @@ -18,7 +18,7 @@ const shim = require('./shim').default; // const Promise = require('promise'); const fs = require('fs-extra'); -function dateToTimestamp(s, zeroIfInvalid = false) { +function dateToTimestamp(s, defaultValue = null) { // Most dates seem to be in this format let m = moment(s, 'YYYYMMDDTHHmmssZ'); @@ -27,7 +27,7 @@ function dateToTimestamp(s, zeroIfInvalid = false) { if (!m.isValid()) m = moment(s, 'YYYYMMDDThmmss AZ'); if (!m.isValid()) { - if (zeroIfInvalid) return 0; + if (defaultValue !== null) return defaultValue; throw new Error(`Invalid date: ${s}`); } @@ -258,6 +258,20 @@ function importEnex(parentFolderId, filePath, importOptions = null) { if (!('onProgress' in importOptions)) importOptions.onProgress = function() {}; if (!('onError' in importOptions)) importOptions.onError = function() {}; + const handleSaxStreamEvent = (fn) => { + return function(...args) { + try { + fn(...args); + } catch (error) { + if (importOptions.onError) { + importOptions.onError(error); + } else { + console.error(error); + } + } + }; + }; + return new Promise((resolve, reject) => { const progressState = { loaded: 0, @@ -337,9 +351,15 @@ function importEnex(parentFolderId, filePath, importOptions = null) { note.parent_id = parentFolderId; note.body = body; - // Notes in enex files always have a created timestamp but not always an - // updated timestamp (it the note has never been modified). For sync - // we require an updated_time property, so set it to create_time in that case + // If the created timestamp was invalid, it would be + // set to zero, so set it to the current date here + if (!note.created_time) note.created_time = Date.now(); + + // Notes in enex files always have a created timestamp + // but not always an updated timestamp (it the note has + // never been modified). For sync we require an + // updated_time property, so set it to create_time in + // that case if (!note.updated_time) note.updated_time = note.created_time; const result = await saveNoteToStorage(note, importOptions); @@ -368,7 +388,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) { importOptions.onError(error); }); - saxStream.on('text', function(text) { + saxStream.on('text', handleSaxStreamEvent(function(text) { const n = currentNodeName(); if (noteAttributes) { @@ -395,9 +415,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) { if (n == 'title') { note.title = text; } else if (n == 'created') { - note.created_time = dateToTimestamp(text); + note.created_time = dateToTimestamp(text, 0); } else if (n == 'updated') { - note.updated_time = dateToTimestamp(text); + note.updated_time = dateToTimestamp(text, 0); } else if (n == 'tag') { note.tags.push(text); } else if (n == 'note') { @@ -408,9 +428,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) { console.warn(`Unsupported note tag: ${n}`); } } - }); + })); - saxStream.on('opentag', function(node) { + saxStream.on('opentag', handleSaxStreamEvent(function(node) { const n = node.name.toLowerCase(); nodes.push(node); @@ -428,9 +448,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) { } else if (n == 'resource') { noteResource = {}; } - }); + })); - saxStream.on('cdata', function(data) { + saxStream.on('cdata', handleSaxStreamEvent(function(data) { const n = currentNodeName(); if (noteResourceRecognition) { @@ -444,9 +464,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) { } } } - }); + })); - saxStream.on('closetag', async function(n) { + saxStream.on('closetag', handleSaxStreamEvent(function(n) { nodes.pop(); if (n == 'note') { @@ -476,9 +496,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) { note.altitude = noteAttributes.altitude; note.author = noteAttributes.author ? noteAttributes.author.trim() : ''; note.is_todo = noteAttributes['reminder-order'] !== '0' && !!noteAttributes['reminder-order']; - note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], true); - note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], true); - note.order = dateToTimestamp(noteAttributes['reminder-order'], true); + note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], 0); + note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], 0); + note.order = dateToTimestamp(noteAttributes['reminder-order'], 0); note.source = noteAttributes.source ? `evernote.${noteAttributes.source.trim()}` : 'evernote'; note.source_url = noteAttributes['source-url'] ? noteAttributes['source-url'].trim() : ''; @@ -495,9 +515,9 @@ function importEnex(parentFolderId, filePath, importOptions = null) { noteResource = null; } - }); + })); - saxStream.on('end', function() { + saxStream.on('end', handleSaxStreamEvent(function() { // Wait till there is no more notes to process. const iid = shim.setInterval(() => { processNotes().then(allDone => { @@ -507,7 +527,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) { } }); }, 500); - }); + })); stream.pipe(saxStream); });