mirror of
https://github.com/laurent22/joplin.git
synced 2024-12-21 09:38:01 +02:00
e11e57f1d8
The implementation uses / symbol as a nesting separator. I.e. tag/subtag is a nested tag, where tag is the parent tag and subtag is its child. Creating a tag named tag/subtag/subsubtag creates three tags, one for each level. The tags are associated using parent_id field. In the app, viewing notes with a tag will also show all notes that are associated with any of the tag's descendant tags (same for the note count). Deleting a tag will also delete all its descendant tags. In the desktop app the tags are shown nested just like the notebooks.
495 lines
14 KiB
JavaScript
495 lines
14 KiB
JavaScript
const { uuid } = require('lib/uuid.js');
|
|
const moment = require('moment');
|
|
const BaseModel = require('lib/BaseModel.js');
|
|
const Note = require('lib/models/Note.js');
|
|
const Tag = require('lib/models/Tag.js');
|
|
const Resource = require('lib/models/Resource.js');
|
|
const Setting = require('lib/models/Setting.js');
|
|
const { MarkupToHtml } = require('lib/joplin-renderer');
|
|
const { enexXmlToMd } = require('./import-enex-md-gen.js');
|
|
const { enexXmlToHtml } = require('./import-enex-html-gen.js');
|
|
const { time } = require('lib/time-utils.js');
|
|
const Levenshtein = require('levenshtein');
|
|
const md5 = require('md5');
|
|
const { Base64Decode } = require('base64-stream');
|
|
const md5File = require('md5-file');
|
|
|
|
// const Promise = require('promise');
|
|
const fs = require('fs-extra');
|
|
|
|
function dateToTimestamp(s, zeroIfInvalid = false) {
|
|
// Most dates seem to be in this format
|
|
let m = moment(s, 'YYYYMMDDTHHmmssZ');
|
|
|
|
// But sometimes they might be in this format eg. 20180306T91108 AMZ
|
|
// https://github.com/laurent22/joplin/issues/557
|
|
if (!m.isValid()) m = moment(s, 'YYYYMMDDThmmss AZ');
|
|
|
|
if (!m.isValid()) {
|
|
if (zeroIfInvalid) return 0;
|
|
throw new Error(`Invalid date: ${s}`);
|
|
}
|
|
|
|
return m.toDate().getTime();
|
|
}
|
|
|
|
function extractRecognitionObjId(recognitionXml) {
|
|
const r = recognitionXml.match(/objID="(.*?)"/);
|
|
return r && r.length >= 2 ? r[1] : null;
|
|
}
|
|
|
|
async function decodeBase64File(sourceFilePath, destFilePath) {
|
|
return new Promise(function(resolve, reject) {
|
|
// Note: we manually handle closing the file so that we can
|
|
// force flusing it before close. This is needed because
|
|
// "end" might be called before the file has been flushed
|
|
// to disk, thus resulting in the calling code to find a
|
|
// file with size 0.
|
|
|
|
const destFile = fs.openSync(destFilePath, 'w');
|
|
const sourceStream = fs.createReadStream(sourceFilePath);
|
|
const destStream = fs.createWriteStream(destFile, {
|
|
fd: destFile,
|
|
autoClose: false,
|
|
});
|
|
sourceStream.pipe(new Base64Decode()).pipe(destStream);
|
|
|
|
sourceStream.on('end', () => {
|
|
fs.fdatasyncSync(destFile);
|
|
fs.closeSync(destFile);
|
|
resolve();
|
|
});
|
|
|
|
sourceStream.on('error', (error) => reject(error));
|
|
});
|
|
}
|
|
|
|
async function md5FileAsync(filePath) {
|
|
return new Promise((resolve, reject) => {
|
|
md5File(filePath, (error, hash) => {
|
|
if (error) {
|
|
reject(error);
|
|
return;
|
|
}
|
|
|
|
resolve(hash);
|
|
});
|
|
});
|
|
}
|
|
|
|
function removeUndefinedProperties(note) {
|
|
const output = {};
|
|
for (const n in note) {
|
|
if (!note.hasOwnProperty(n)) continue;
|
|
const v = note[n];
|
|
if (v === undefined || v === null) continue;
|
|
output[n] = v;
|
|
}
|
|
return output;
|
|
}
|
|
|
|
function levenshteinPercent(s1, s2) {
|
|
const l = new Levenshtein(s1, s2);
|
|
if (!s1.length || !s2.length) return 1;
|
|
return Math.abs(l.distance / s1.length);
|
|
}
|
|
|
|
async function fuzzyMatch(note) {
|
|
if (note.created_time < time.unixMs() - 1000 * 60 * 60 * 24 * 360) {
|
|
const notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ? AND title = ?', [note.created_time, note.title]);
|
|
return notes.length !== 1 ? null : notes[0];
|
|
}
|
|
|
|
const notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', [note.created_time]);
|
|
if (notes.length === 0) return null;
|
|
if (notes.length === 1) return notes[0];
|
|
|
|
let lowestL = 1;
|
|
let lowestN = null;
|
|
for (let i = 0; i < notes.length; i++) {
|
|
const n = notes[i];
|
|
const l = levenshteinPercent(note.title, n.title);
|
|
if (l < lowestL) {
|
|
lowestL = l;
|
|
lowestN = n;
|
|
}
|
|
}
|
|
|
|
if (lowestN && lowestL < 0.2) return lowestN;
|
|
|
|
return null;
|
|
}
|
|
|
|
// At this point we have the resource has it's been parsed from the XML, but additional
|
|
// processing needs to be done to get the final resource file, its size, MD5, etc.
|
|
async function processNoteResource(resource) {
|
|
if (resource.dataEncoding == 'base64') {
|
|
const decodedFilePath = `${resource.dataFilePath}.decoded`;
|
|
await decodeBase64File(resource.dataFilePath, decodedFilePath);
|
|
resource.dataFilePath = decodedFilePath;
|
|
} else if (resource.dataEncoding) {
|
|
throw new Error(`Cannot decode resource with encoding: ${resource.dataEncoding}`);
|
|
}
|
|
|
|
const stats = fs.statSync(resource.dataFilePath);
|
|
resource.size = stats.size;
|
|
|
|
if (!resource.id) {
|
|
// If no resource ID is present, the resource ID is actually the MD5 of the data.
|
|
// This ID will match the "hash" attribute of the corresponding <en-media> tag.
|
|
// resourceId = md5(decodedData);
|
|
resource.id = await md5FileAsync(resource.dataFilePath);
|
|
}
|
|
|
|
if (!resource.id || !resource.size) {
|
|
const debugTemp = Object.assign({}, resource);
|
|
debugTemp.data = debugTemp.data ? `${debugTemp.data.substr(0, 32)}...` : debugTemp.data;
|
|
throw new Error(`This resource was not added because it has no ID or no content: ${JSON.stringify(debugTemp)}`);
|
|
}
|
|
|
|
return resource;
|
|
}
|
|
|
|
async function saveNoteResources(note, importOptions) {
|
|
let resourcesCreated = 0;
|
|
for (let i = 0; i < note.resources.length; i++) {
|
|
let resource = note.resources[i];
|
|
|
|
try {
|
|
resource = await processNoteResource(resource);
|
|
} catch (error) {
|
|
importOptions.onError(error);
|
|
continue;
|
|
}
|
|
|
|
const toSave = Object.assign({}, resource);
|
|
delete toSave.dataFilePath;
|
|
delete toSave.dataEncoding;
|
|
|
|
// The same resource sometimes appear twice in the same enex (exact same ID and file).
|
|
// In that case, just skip it - it means two different notes might be linked to the
|
|
// same resource.
|
|
const existingResource = await Resource.load(toSave.id);
|
|
if (existingResource) continue;
|
|
|
|
await fs.move(resource.dataFilePath, Resource.fullPath(toSave), { overwrite: true });
|
|
await Resource.save(toSave, { isNew: true });
|
|
resourcesCreated++;
|
|
}
|
|
return resourcesCreated;
|
|
}
|
|
|
|
async function saveNoteTags(note) {
|
|
let notesTagged = 0;
|
|
for (let i = 0; i < note.tags.length; i++) {
|
|
const tagTitle = note.tags[i];
|
|
|
|
let tag = await Tag.loadByTitle(tagTitle);
|
|
if (!tag) tag = await Tag.saveNested({}, tagTitle);
|
|
|
|
await Tag.addNote(tag.id, note.id);
|
|
|
|
notesTagged++;
|
|
}
|
|
return notesTagged;
|
|
}
|
|
|
|
async function saveNoteToStorage(note, importOptions) {
|
|
importOptions = Object.assign({}, {
|
|
fuzzyMatching: false,
|
|
}, importOptions);
|
|
|
|
note = Note.filter(note);
|
|
|
|
const existingNote = importOptions.fuzzyMatching ? await fuzzyMatch(note) : null;
|
|
|
|
const result = {
|
|
noteCreated: false,
|
|
noteUpdated: false,
|
|
noteSkipped: false,
|
|
resourcesCreated: 0,
|
|
notesTagged: 0,
|
|
};
|
|
|
|
const resourcesCreated = await saveNoteResources(note, importOptions);
|
|
result.resourcesCreated += resourcesCreated;
|
|
|
|
const notesTagged = await saveNoteTags(note);
|
|
result.notesTagged += notesTagged;
|
|
|
|
if (existingNote) {
|
|
const diff = BaseModel.diffObjects(existingNote, note);
|
|
delete diff.tags;
|
|
delete diff.resources;
|
|
delete diff.id;
|
|
|
|
if (!Object.getOwnPropertyNames(diff).length) {
|
|
result.noteSkipped = true;
|
|
return result;
|
|
}
|
|
|
|
diff.id = existingNote.id;
|
|
diff.type_ = existingNote.type_;
|
|
await Note.save(diff, { autoTimestamp: false });
|
|
result.noteUpdated = true;
|
|
} else {
|
|
await Note.save(note, {
|
|
isNew: true,
|
|
autoTimestamp: false,
|
|
});
|
|
result.noteCreated = true;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function importEnex(parentFolderId, filePath, importOptions = null) {
|
|
if (!importOptions) importOptions = {};
|
|
// console.info(JSON.stringify({importOptions}, null, 2));
|
|
if (!('fuzzyMatching' in importOptions)) importOptions.fuzzyMatching = false;
|
|
if (!('onProgress' in importOptions)) importOptions.onProgress = function() {};
|
|
if (!('onError' in importOptions)) importOptions.onError = function() {};
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const progressState = {
|
|
loaded: 0,
|
|
created: 0,
|
|
updated: 0,
|
|
skipped: 0,
|
|
resourcesCreated: 0,
|
|
notesTagged: 0,
|
|
};
|
|
|
|
const stream = fs.createReadStream(filePath);
|
|
|
|
const options = {};
|
|
const strict = true;
|
|
const saxStream = require('sax').createStream(strict, options);
|
|
|
|
const nodes = []; // LIFO list of nodes so that we know in which node we are in the onText event
|
|
let note = null;
|
|
let noteAttributes = null;
|
|
let noteResource = null;
|
|
let noteResourceAttributes = null;
|
|
let noteResourceRecognition = null;
|
|
const notes = [];
|
|
let processingNotes = false;
|
|
|
|
stream.on('error', error => {
|
|
reject(new Error(error.toString()));
|
|
});
|
|
|
|
function currentNodeName() {
|
|
if (!nodes.length) return null;
|
|
return nodes[nodes.length - 1].name;
|
|
}
|
|
|
|
function currentNodeAttributes() {
|
|
if (!nodes.length) return {};
|
|
return nodes[nodes.length - 1].attributes;
|
|
}
|
|
|
|
async function processNotes() {
|
|
if (processingNotes) return false;
|
|
|
|
try {
|
|
processingNotes = true;
|
|
stream.pause();
|
|
|
|
while (notes.length) {
|
|
const note = notes.shift();
|
|
const body = importOptions.outputFormat === 'html' ?
|
|
await enexXmlToHtml(note.bodyXml, note.resources) :
|
|
await enexXmlToMd(note.bodyXml, note.resources);
|
|
delete note.bodyXml;
|
|
|
|
note.markup_language = importOptions.outputFormat === 'html' ?
|
|
MarkupToHtml.MARKUP_LANGUAGE_HTML :
|
|
MarkupToHtml.MARKUP_LANGUAGE_MARKDOWN;
|
|
|
|
// console.info('*************************************************************************');
|
|
// console.info(body);
|
|
// console.info('*************************************************************************');
|
|
|
|
note.id = uuid.create();
|
|
note.parent_id = parentFolderId;
|
|
note.body = body;
|
|
|
|
// Notes in enex files always have a created timestamp but not always an
|
|
// updated timestamp (it the note has never been modified). For sync
|
|
// we require an updated_time property, so set it to create_time in that case
|
|
if (!note.updated_time) note.updated_time = note.created_time;
|
|
|
|
const result = await saveNoteToStorage(note, importOptions);
|
|
|
|
if (result.noteUpdated) {
|
|
progressState.updated++;
|
|
} else if (result.noteCreated) {
|
|
progressState.created++;
|
|
} else if (result.noteSkipped) {
|
|
progressState.skipped++;
|
|
}
|
|
progressState.resourcesCreated += result.resourcesCreated;
|
|
progressState.notesTagged += result.notesTagged;
|
|
importOptions.onProgress(progressState);
|
|
}
|
|
} catch (error) {
|
|
console.error(error);
|
|
}
|
|
|
|
stream.resume();
|
|
processingNotes = false;
|
|
return true;
|
|
}
|
|
|
|
saxStream.on('error', error => {
|
|
importOptions.onError(error);
|
|
});
|
|
|
|
saxStream.on('text', function(text) {
|
|
const n = currentNodeName();
|
|
|
|
if (noteAttributes) {
|
|
noteAttributes[n] = text;
|
|
} else if (noteResourceAttributes) {
|
|
noteResourceAttributes[n] = text;
|
|
} else if (noteResource) {
|
|
if (n == 'data') {
|
|
if (!noteResource.dataEncoding) {
|
|
const attr = currentNodeAttributes();
|
|
noteResource.dataEncoding = attr.encoding;
|
|
}
|
|
|
|
if (!noteResource.dataFilePath) {
|
|
noteResource.dataFilePath = `${Setting.value('tempDir')}/${md5(Date.now() + Math.random())}.base64`;
|
|
}
|
|
|
|
fs.appendFileSync(noteResource.dataFilePath, text);
|
|
} else {
|
|
if (!(n in noteResource)) noteResource[n] = '';
|
|
noteResource[n] += text;
|
|
}
|
|
} else if (note) {
|
|
if (n == 'title') {
|
|
note.title = text;
|
|
} else if (n == 'created') {
|
|
note.created_time = dateToTimestamp(text);
|
|
} else if (n == 'updated') {
|
|
note.updated_time = dateToTimestamp(text);
|
|
} else if (n == 'tag') {
|
|
note.tags.push(text);
|
|
} else if (n == 'note') {
|
|
// Ignore - white space between the opening tag <note> and the first sub-tag
|
|
} else if (n == 'content') {
|
|
// Ignore - white space between the opening tag <content> and the <![CDATA[< block where the content actually is
|
|
} else {
|
|
console.warn(`Unsupported note tag: ${n}`);
|
|
}
|
|
}
|
|
});
|
|
|
|
saxStream.on('opentag', function(node) {
|
|
const n = node.name.toLowerCase();
|
|
nodes.push(node);
|
|
|
|
if (n == 'note') {
|
|
note = {
|
|
resources: [],
|
|
tags: [],
|
|
};
|
|
} else if (n == 'resource-attributes') {
|
|
noteResourceAttributes = {};
|
|
} else if (n == 'recognition') {
|
|
if (noteResource) noteResourceRecognition = {};
|
|
} else if (n == 'note-attributes') {
|
|
noteAttributes = {};
|
|
} else if (n == 'resource') {
|
|
noteResource = {};
|
|
}
|
|
});
|
|
|
|
saxStream.on('cdata', function(data) {
|
|
const n = currentNodeName();
|
|
|
|
if (noteResourceRecognition) {
|
|
noteResourceRecognition.objID = extractRecognitionObjId(data);
|
|
} else if (note) {
|
|
if (n == 'content') {
|
|
if ('bodyXml' in note) {
|
|
note.bodyXml += data;
|
|
} else {
|
|
note.bodyXml = data;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
saxStream.on('closetag', async function(n) {
|
|
nodes.pop();
|
|
|
|
if (n == 'note') {
|
|
note = removeUndefinedProperties(note);
|
|
|
|
progressState.loaded++;
|
|
importOptions.onProgress(progressState);
|
|
|
|
notes.push(note);
|
|
|
|
if (notes.length >= 10) {
|
|
processNotes().catch(error => {
|
|
importOptions.onError(error);
|
|
});
|
|
}
|
|
note = null;
|
|
} else if (n == 'recognition' && noteResource) {
|
|
noteResource.id = noteResourceRecognition.objID;
|
|
noteResourceRecognition = null;
|
|
} else if (n == 'resource-attributes') {
|
|
noteResource.filename = noteResourceAttributes['file-name'];
|
|
if (noteResourceAttributes['source-url']) noteResource.sourceUrl = noteResourceAttributes['source-url'];
|
|
noteResourceAttributes = null;
|
|
} else if (n == 'note-attributes') {
|
|
note.latitude = noteAttributes.latitude;
|
|
note.longitude = noteAttributes.longitude;
|
|
note.altitude = noteAttributes.altitude;
|
|
note.author = noteAttributes.author;
|
|
note.is_todo = noteAttributes['reminder-order'] !== '0' && !!noteAttributes['reminder-order'];
|
|
note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], true);
|
|
note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], true);
|
|
note.order = dateToTimestamp(noteAttributes['reminder-order'], true);
|
|
note.source = noteAttributes.source ? `evernote.${noteAttributes.source}` : 'evernote';
|
|
note.source_url = noteAttributes['source-url'] ? noteAttributes['source-url'] : '';
|
|
|
|
noteAttributes = null;
|
|
} else if (n == 'resource') {
|
|
note.resources.push({
|
|
id: noteResource.id,
|
|
dataFilePath: noteResource.dataFilePath,
|
|
dataEncoding: noteResource.dataEncoding,
|
|
mime: noteResource.mime,
|
|
title: noteResource.filename ? noteResource.filename : '',
|
|
filename: noteResource.filename ? noteResource.filename : '',
|
|
});
|
|
|
|
noteResource = null;
|
|
}
|
|
});
|
|
|
|
saxStream.on('end', function() {
|
|
// Wait till there is no more notes to process.
|
|
const iid = setInterval(() => {
|
|
processNotes().then(allDone => {
|
|
if (allDone) {
|
|
clearTimeout(iid);
|
|
resolve();
|
|
}
|
|
});
|
|
}, 500);
|
|
});
|
|
|
|
stream.pipe(saxStream);
|
|
});
|
|
}
|
|
|
|
module.exports = { importEnex };
|