1
0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-03 08:35:29 +02:00
joplin/ReactNativeClient/lib/import-enex.js

432 lines
12 KiB
JavaScript
Raw Normal View History

const { uuid } = require('lib/uuid.js');
const moment = require('moment');
const { promiseChain } = require('lib/promise-utils.js');
const { folderItemFilename } = require('lib/string-utils.js');
const BaseModel = require('lib/BaseModel.js');
const Note = require('lib/models/Note.js');
const Tag = require('lib/models/Tag.js');
const Resource = require('lib/models/Resource.js');
const Folder = require('lib/models/Folder.js');
const { enexXmlToMd } = require('./import-enex-md-gen.js');
const { time } = require('lib/time-utils.js');
const Levenshtein = require('levenshtein');
const jsSHA = require("jssha");
const md5 = require('md5');
2017-06-06 23:40:09 +02:00
2017-11-11 19:36:47 +02:00
//const Promise = require('promise');
const fs = require('fs-extra');
2017-06-24 19:40:03 +02:00
2017-06-25 01:19:11 +02:00
function dateToTimestamp(s, zeroIfInvalid = false) {
// Most dates seem to be in this format
let m = moment(s, 'YYYYMMDDTHHmmssZ');
// But sometimes they might be in this format eg. 20180306T91108 AMZ
// https://github.com/laurent22/joplin/issues/557
if (!m.isValid()) m = moment(s, 'YYYYMMDDThmmss AZ');
2017-06-06 23:40:09 +02:00
if (!m.isValid()) {
2017-06-25 01:19:11 +02:00
if (zeroIfInvalid) return 0;
throw new Error('Invalid date: ' + s);
2017-06-06 23:27:03 +02:00
}
2017-06-24 19:40:03 +02:00
return m.toDate().getTime();
2017-06-06 23:40:09 +02:00
}
2017-06-09 00:24:40 +02:00
function extractRecognitionObjId(recognitionXml) {
const r = recognitionXml.match(/objID="(.*?)"/);
return r && r.length >= 2 ? r[1] : null;
}
2017-06-06 23:40:09 +02:00
async function filePutContents(filePath, content) {
await fs.writeFile(filePath, content);
2017-06-11 23:11:14 +02:00
}
2017-06-24 19:40:03 +02:00
function removeUndefinedProperties(note) {
let output = {};
for (let n in note) {
if (!note.hasOwnProperty(n)) continue;
let v = note[n];
if (v === undefined || v === null) continue;
output[n] = v;
}
return output;
2017-06-11 23:11:14 +02:00
}
2017-06-09 00:24:40 +02:00
function createNoteId(note) {
let shaObj = new jsSHA("SHA-256", "TEXT");
shaObj.update(note.title + '_' + note.body + "_" + note.created_time + "_" + note.updated_time + "_");
2017-06-09 00:24:40 +02:00
let hash = shaObj.getHash("HEX");
return hash.substr(0, 32);
}
2017-06-06 23:40:09 +02:00
2017-07-12 22:39:47 +02:00
function levenshteinPercent(s1, s2) {
let l = new Levenshtein(s1, s2);
if (!s1.length || !s2.length) return 1;
return Math.abs(l.distance / s1.length);
}
2017-06-24 19:40:03 +02:00
async function fuzzyMatch(note) {
2017-07-12 22:39:47 +02:00
if (note.created_time < time.unixMs() - 1000 * 60 * 60 * 24 * 360) {
let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ? AND title = ?', [note.created_time, note.title]);
2017-07-12 22:39:47 +02:00
return notes.length !== 1 ? null : notes[0];
}
let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', [note.created_time]);
2017-07-12 22:39:47 +02:00
if (notes.length === 0) return null;
if (notes.length === 1) return notes[0];
let lowestL = 1;
let lowestN = null;
for (let i = 0; i < notes.length; i++) {
let n = notes[i];
let l = levenshteinPercent(note.title, n.title);
if (l < lowestL) {
lowestL = l;
lowestN = n;
}
}
if (lowestN && lowestL < 0.2) return lowestN;
return null;
2017-06-24 19:40:03 +02:00
}
2017-06-27 22:53:40 +02:00
async function saveNoteResources(note) {
let resourcesCreated = 0;
for (let i = 0; i < note.resources.length; i++) {
let resource = note.resources[i];
let toSave = Object.assign({}, resource);
delete toSave.data;
// The same resource sometimes appear twice in the same enex (exact same ID and file).
// In that case, just skip it - it means two different notes might be linked to the
// same resource.
let existingResource = await Resource.load(toSave.id);
if (existingResource) continue;
await filePutContents(Resource.fullPath(toSave), resource.data)
2017-07-19 21:15:55 +02:00
await Resource.save(toSave, { isNew: true });
2017-06-27 22:53:40 +02:00
resourcesCreated++;
}
return resourcesCreated;
}
2017-07-02 17:46:03 +02:00
async function saveNoteTags(note) {
2017-07-02 20:38:34 +02:00
let notesTagged = 0;
2017-07-02 17:46:03 +02:00
for (let i = 0; i < note.tags.length; i++) {
let tagTitle = note.tags[i];
let tag = await Tag.loadByTitle(tagTitle);
if (!tag) tag = await Tag.save({ title: tagTitle });
await Tag.addNote(tag.id, note.id);
2017-07-02 20:38:34 +02:00
notesTagged++;
2017-07-02 17:46:03 +02:00
}
2017-07-02 20:38:34 +02:00
return notesTagged;
2017-07-02 17:46:03 +02:00
}
2017-06-25 13:39:42 +02:00
async function saveNoteToStorage(note, fuzzyMatching = false) {
2017-06-24 19:40:03 +02:00
note = Note.filter(note);
2017-06-25 13:39:42 +02:00
let existingNote = fuzzyMatching ? await fuzzyMatch(note) : null;
2017-06-24 19:40:03 +02:00
2017-06-26 21:12:49 +02:00
let result = {
noteCreated: false,
noteUpdated: false,
2017-06-27 01:20:01 +02:00
noteSkipped: false,
2017-06-26 21:12:49 +02:00
resourcesCreated: 0,
2017-07-02 20:38:34 +02:00
notesTagged: 0,
2017-06-26 21:12:49 +02:00
};
2017-06-27 22:53:40 +02:00
let resourcesCreated = await saveNoteResources(note);
result.resourcesCreated += resourcesCreated;
2017-07-02 20:38:34 +02:00
let notesTagged = await saveNoteTags(note);
result.notesTagged += notesTagged;
2017-07-02 17:46:03 +02:00
2017-06-24 19:40:03 +02:00
if (existingNote) {
let diff = BaseModel.diffObjects(existingNote, note);
delete diff.tags;
delete diff.resources;
delete diff.id;
2017-06-27 01:20:01 +02:00
if (!Object.getOwnPropertyNames(diff).length) {
result.noteSkipped = true;
return result;
}
2017-06-24 19:40:03 +02:00
diff.id = existingNote.id;
diff.type_ = existingNote.type_;
await Note.save(diff, { autoTimestamp: false })
2017-06-27 01:20:01 +02:00
result.noteUpdated = true;
2017-06-24 19:40:03 +02:00
} else {
2017-06-27 01:20:01 +02:00
await Note.save(note, {
2017-06-24 20:51:43 +02:00
isNew: true,
autoTimestamp: false,
});
2017-06-27 01:20:01 +02:00
result.noteCreated = true;
2017-06-24 19:40:03 +02:00
}
2017-06-27 01:20:01 +02:00
return result;
2017-06-24 19:40:03 +02:00
}
2017-06-25 13:39:42 +02:00
function importEnex(parentFolderId, filePath, importOptions = null) {
if (!importOptions) importOptions = {};
if (!('fuzzyMatching' in importOptions)) importOptions.fuzzyMatching = false;
if (!('onProgress' in importOptions)) importOptions.onProgress = function(state) {};
if (!('onError' in importOptions)) importOptions.onError = function(error) {};
2017-06-24 19:40:03 +02:00
2017-06-09 00:24:40 +02:00
return new Promise((resolve, reject) => {
let progressState = {
loaded: 0,
2017-06-26 21:12:49 +02:00
created: 0,
updated: 0,
2017-06-27 01:20:01 +02:00
skipped: 0,
2017-06-26 21:12:49 +02:00
resourcesCreated: 0,
2017-07-02 20:38:34 +02:00
notesTagged: 0,
};
let stream = fs.createReadStream(filePath);
2017-06-09 00:24:40 +02:00
let options = {};
let strict = true;
let saxStream = require('sax').createStream(strict, options);
2017-06-09 00:24:40 +02:00
let nodes = []; // LIFO list of nodes so that we know in which node we are in the onText event
let note = null;
let noteAttributes = null;
let noteResource = null;
let noteResourceAttributes = null;
let noteResourceRecognition = null;
let notes = [];
let processingNotes = false;
2017-06-09 00:24:40 +02:00
stream.on('error', (error) => {
2017-06-29 23:04:11 +02:00
reject(new Error(error.toString()));
2017-06-24 19:40:03 +02:00
});
2017-06-09 00:24:40 +02:00
function currentNodeName() {
if (!nodes.length) return null;
return nodes[nodes.length - 1].name;
}
2017-06-06 23:40:09 +02:00
2017-06-09 00:24:40 +02:00
function currentNodeAttributes() {
if (!nodes.length) return {};
return nodes[nodes.length - 1].attributes;
}
2017-06-06 23:40:09 +02:00
async function processNotes() {
2017-06-27 01:20:01 +02:00
if (processingNotes) return false;
try {
processingNotes = true;
stream.pause();
while (notes.length) {
let note = notes.shift();
const body = await enexXmlToMd(note.bodyXml, note.resources);
delete note.bodyXml;
// console.info('*************************************************************************');
// console.info(body);
// console.info('*************************************************************************');
note.id = uuid.create();
note.parent_id = parentFolderId;
note.body = body;
// Notes in enex files always have a created timestamp but not always an
// updated timestamp (it the note has never been modified). For sync
// we require an updated_time property, so set it to create_time in that case
if (!note.updated_time) note.updated_time = note.created_time;
const result = await saveNoteToStorage(note, importOptions.fuzzyMatching);
if (result.noteUpdated) {
progressState.updated++;
} else if (result.noteCreated) {
progressState.created++;
} else if (result.noteSkipped) {
progressState.skipped++;
}
progressState.resourcesCreated += result.resourcesCreated;
progressState.notesTagged += result.notesTagged;
importOptions.onProgress(progressState);
}
} catch(error) {
console.error(error);
2017-06-09 00:24:40 +02:00
}
2017-06-06 23:40:09 +02:00
stream.resume();
processingNotes = false;
return true;
2017-06-09 00:24:40 +02:00
}
2017-06-07 21:21:04 +02:00
saxStream.on('error', (error) => {
importOptions.onError(error);
});
2017-06-07 21:21:04 +02:00
saxStream.on('text', function(text) {
2017-06-09 00:24:40 +02:00
let n = currentNodeName();
if (noteAttributes) {
noteAttributes[n] = text;
} else if (noteResourceAttributes) {
noteResourceAttributes[n] = text;
2017-06-09 00:24:40 +02:00
} else if (noteResource) {
if (n == 'data') {
2017-06-09 00:24:40 +02:00
let attr = currentNodeAttributes();
noteResource.dataEncoding = attr.encoding;
}
if (!(n in noteResource)) noteResource[n] = '';
noteResource[n] += text;
2017-06-09 00:24:40 +02:00
} else if (note) {
if (n == 'title') {
2017-06-09 00:24:40 +02:00
note.title = text;
} else if (n == 'created') {
note.created_time = dateToTimestamp(text);
} else if (n == 'updated') {
2017-06-09 00:24:40 +02:00
note.updated_time = dateToTimestamp(text);
} else if (n == 'tag') {
2017-06-09 00:24:40 +02:00
note.tags.push(text);
} else if (n == 'note') {
// Ignore - white space between the opening tag <note> and the first sub-tag
} else if (n == 'content') {
// Ignore - white space between the opening tag <content> and the <![CDATA[< block where the content actually is
2017-06-25 01:19:11 +02:00
} else {
console.warn('Unsupported note tag: ' + n);
2017-06-09 00:24:40 +02:00
}
}
})
2017-06-06 23:40:09 +02:00
saxStream.on('opentag', function(node) {
2017-06-09 00:24:40 +02:00
let n = node.name.toLowerCase();
nodes.push(node);
if (n == 'note') {
2017-06-09 00:24:40 +02:00
note = {
resources: [],
tags: [],
};
} else if (n == 'resource-attributes') {
2017-06-09 00:24:40 +02:00
noteResourceAttributes = {};
} else if (n == 'recognition') {
2017-06-09 00:24:40 +02:00
if (noteResource) noteResourceRecognition = {};
} else if (n == 'note-attributes') {
2017-06-09 00:24:40 +02:00
noteAttributes = {};
} else if (n == 'resource') {
2017-06-09 00:24:40 +02:00
noteResource = {};
}
});
2017-06-06 23:40:09 +02:00
saxStream.on('cdata', function(data) {
2017-06-09 00:24:40 +02:00
let n = currentNodeName();
2017-06-06 23:40:09 +02:00
2017-06-09 00:24:40 +02:00
if (noteResourceRecognition) {
noteResourceRecognition.objID = extractRecognitionObjId(data);
} else if (note) {
if (n == 'content') {
if ('bodyXml' in note) {
note.bodyXml += data;
} else {
note.bodyXml = data;
}
2017-06-09 00:24:40 +02:00
}
}
});
2017-06-06 23:27:03 +02:00
saxStream.on('closetag', function(n) {
2017-06-09 00:24:40 +02:00
nodes.pop();
if (n == 'note') {
2017-06-24 19:40:03 +02:00
note = removeUndefinedProperties(note);
progressState.loaded++;
importOptions.onProgress(progressState);
2017-06-09 00:24:40 +02:00
notes.push(note);
2017-06-09 00:24:40 +02:00
if (notes.length >= 10) {
processNotes().catch((error) => {
importOptions.onError(error);
2017-06-09 00:24:40 +02:00
});
}
note = null;
} else if (n == 'recognition' && noteResource) {
2017-06-09 00:24:40 +02:00
noteResource.id = noteResourceRecognition.objID;
noteResourceRecognition = null;
} else if (n == 'resource-attributes') {
noteResource.filename = noteResourceAttributes['file-name'];
2017-06-09 00:24:40 +02:00
noteResourceAttributes = null;
} else if (n == 'note-attributes') {
2017-06-09 00:24:40 +02:00
note.latitude = noteAttributes.latitude;
note.longitude = noteAttributes.longitude;
note.altitude = noteAttributes.altitude;
note.author = noteAttributes.author;
note.is_todo = !!noteAttributes['reminder-order'];
note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], true);
note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], true);
note.order = dateToTimestamp(noteAttributes['reminder-order'], true);
note.source = !!noteAttributes.source ? 'evernote.' + noteAttributes.source : 'evernote';
note.source_url = !!noteAttributes['source-url'] ? noteAttributes['source-url'] : '';
2017-06-25 01:19:11 +02:00
2017-06-26 00:47:54 +02:00
// if (noteAttributes['reminder-time']) {
// console.info('======================================================');
// console.info(noteAttributes);
// console.info('------------------------------------------------------');
// console.info(note);
// console.info('======================================================');
// }
2017-06-25 01:19:11 +02:00
2017-06-09 00:24:40 +02:00
noteAttributes = null;
} else if (n == 'resource') {
2017-06-09 00:24:40 +02:00
let decodedData = null;
let resourceId = noteResource.id;
if (noteResource.dataEncoding == 'base64') {
try {
decodedData = Buffer.from(noteResource.data, 'base64');
} catch (error) {
importOptions.onError(error);
}
2017-06-09 00:24:40 +02:00
} else {
importOptions.onError(new Error('Cannot decode resource with encoding: ' + noteResource.dataEncoding));
decodedData = noteResource.data; // Just put the encoded data directly in the file so it can, potentially, be manually decoded later
2017-06-09 00:24:40 +02:00
}
2017-06-06 23:27:03 +02:00
if (!resourceId && decodedData) {
// If no resource ID is present, the resource ID is actually the MD5 of the data.
// This ID will match the "hash" attribute of the corresponding <en-media> tag.
resourceId = md5(decodedData);
}
2017-06-09 00:24:40 +02:00
let r = {
id: resourceId,
2017-06-09 00:24:40 +02:00
data: decodedData,
2017-06-24 20:51:43 +02:00
mime: noteResource.mime,
title: noteResource.filename ? noteResource.filename : '',
filename: noteResource.filename ? noteResource.filename : '',
2017-06-09 00:24:40 +02:00
};
2017-06-06 23:27:03 +02:00
2017-06-09 00:24:40 +02:00
note.resources.push(r);
noteResource = null;
}
});
2017-06-06 23:27:03 +02:00
saxStream.on('end', function() {
// Wait till there is no more notes to process.
let iid = setInterval(() => {
processNotes().then((allDone) => {
2017-06-27 01:20:01 +02:00
if (allDone) {
clearTimeout(iid);
resolve();
}
});
}, 500);
2017-06-09 00:24:40 +02:00
});
2017-06-06 23:27:03 +02:00
2017-06-09 00:24:40 +02:00
stream.pipe(saxStream);
});
}
2017-06-06 23:27:03 +02:00
module.exports = { importEnex };