From 4178d1f1de46cae0bcb9f54644de9a5c2017e074 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Tue, 27 Jun 2017 00:20:01 +0100 Subject: [PATCH] Improved enex import --- CliClient/app/import-enex.js | 84 +++++++----- CliClient/app/main.js | 246 +++++++++++++++++++---------------- CliClient/package.json | 2 +- lib/base-model.js | 12 +- lib/database.js | 30 ++++- lib/models/note.js | 2 +- lib/onedrive-api.js | 3 +- lib/time-utils.js | 12 ++ 8 files changed, 232 insertions(+), 159 deletions(-) diff --git a/CliClient/app/import-enex.js b/CliClient/app/import-enex.js index e91e3dfadc..4da3d4ffc4 100644 --- a/CliClient/app/import-enex.js +++ b/CliClient/app/import-enex.js @@ -12,25 +12,6 @@ import jsSHA from "jssha"; const Promise = require('promise'); const fs = require('fs-extra'); const stringToStream = require('string-to-stream') - -let existingTimestamps = []; - -function uniqueCreatedTimestamp(timestamp) { - if (existingTimestamps.indexOf(timestamp) < 0) { - existingTimestamps.push(timestamp); - return timestamp; - } - - for (let i = 1; i <= 999; i++) { - let t = timestamp + i; - if (existingTimestamps.indexOf(t) < 0) { - existingTimestamps.push(t); - return t; - } - } - - return timestamp; -} function dateToTimestamp(s, zeroIfInvalid = false) { let m = moment(s, 'YYYYMMDDTHHmmssZ'); @@ -69,7 +50,7 @@ function createNoteId(note) { } async function fuzzyMatch(note) { - let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', note.created_time); + let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', [note.created_time]); if (!notes.length) return null; if (notes.length === 1) return notes[0]; @@ -92,6 +73,7 @@ async function saveNoteToStorage(note, fuzzyMatching = false) { let result = { noteCreated: false, noteUpdated: false, + noteSkipped: false, resourcesCreated: 0, }; @@ -103,14 +85,16 @@ async function saveNoteToStorage(note, fuzzyMatching = false) { // TODO: also save resources - if (!Object.getOwnPropertyNames(diff).length) return; + if (!Object.getOwnPropertyNames(diff).length) { + result.noteSkipped = true; + // TODO: also save resources + return result; + } diff.id = existingNote.id; diff.type_ = existingNote.type_; - return Note.save(diff, { autoTimestamp: false }).then(() => { - result.noteUpdated = true; - return result; - }); + await Note.save(diff, { autoTimestamp: false }) + result.noteUpdated = true; } else { for (let i = 0; i < note.resources.length; i++) { let resource = note.resources[i]; @@ -128,14 +112,14 @@ async function saveNoteToStorage(note, fuzzyMatching = false) { result.resourcesCreated++; } - return Note.save(note, { + await Note.save(note, { isNew: true, autoTimestamp: false, - }).then(() => { - result.noteCreated = true; - return result; }); + result.noteCreated = true; } + + return result; } function importEnex(parentFolderId, filePath, importOptions = null) { @@ -144,11 +128,35 @@ function importEnex(parentFolderId, filePath, importOptions = null) { if (!('onProgress' in importOptions)) importOptions.onProgress = function(state) {}; if (!('onError' in importOptions)) importOptions.onError = function(error) {}; + // Some notes were created with the exact same timestamp, for example when they were + // batch imported. In order to make fuzzy matching easier, this function ensures + // that each timestamp is unique. + let existingTimestamps = []; + function uniqueCreatedTimestamp(timestamp) { + return timestamp; + + if (existingTimestamps.indexOf(timestamp) < 0) { + existingTimestamps.push(timestamp); + return timestamp; + } + + for (let i = 1; i <= 999; i++) { + let t = timestamp + i; + if (existingTimestamps.indexOf(t) < 0) { + existingTimestamps.push(t); + return t; + } + } + + return timestamp; + } + return new Promise((resolve, reject) => { let progressState = { loaded: 0, created: 0, updated: 0, + skipped: 0, resourcesCreated: 0, }; @@ -182,7 +190,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) { } async function processNotes() { - if (processingNotes) return; + if (processingNotes) return false; + processingNotes = true; stream.pause(); @@ -204,6 +213,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) { progressState.updated++; } else if (result.noteCreated) { progressState.created++; + } else if (result.noteSkipped) { + progressState.skipped++; } progressState.resourcesCreated += result.resourcesCreated; importOptions.onProgress(progressState); @@ -214,6 +225,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) { return promiseChain(chain).then(() => { stream.resume(); processingNotes = false; + return true; }); } @@ -354,12 +366,12 @@ function importEnex(parentFolderId, filePath, importOptions = null) { saxStream.on('end', function() { // Wait till there is no more notes to process. let iid = setInterval(() => { - if (notes.length) { - processNotes(); - } else { - clearInterval(iid); - resolve(); - } + processNotes().then((allDone) => { + if (allDone) { + clearTimeout(iid); + resolve(); + } + }); }, 500); }); diff --git a/CliClient/app/main.js b/CliClient/app/main.js index a09fb5411f..c9fff95814 100644 --- a/CliClient/app/main.js +++ b/CliClient/app/main.js @@ -63,10 +63,11 @@ commands.push({ aliases: ['mkdir'], description: 'Creates a new notebook', action: function(args, end) { - Folder.save({ title: args['notebook'] }).catch((error) => { - this.log(error); - }).then((folder) => { + Folder.save({ title: args['notebook'] }).then((folder) => { switchCurrentFolder(folder); + }).catch((error) => { + this.log(error); + }).then(() => { end(); }); }, @@ -187,34 +188,38 @@ commands.push({ usage: 'rm ', description: 'Deletes the given item. For a notebook, all the notes within that notebook will be deleted. Use `rm ../` to delete a notebook.', action: async function(args, end) { - let pattern = args['pattern']; - let itemType = null; + try { + let pattern = args['pattern']; + let itemType = null; - if (pattern.indexOf('*') < 0) { // Handle it as a simple title - if (pattern.substr(0, 3) == '../') { - itemType = BaseModel.MODEL_TYPE_FOLDER; - pattern = pattern.substr(3); - } else { - itemType = BaseModel.MODEL_TYPE_NOTE; - } + if (pattern.indexOf('*') < 0) { // Handle it as a simple title + if (pattern.substr(0, 3) == '../') { + itemType = BaseModel.MODEL_TYPE_FOLDER; + pattern = pattern.substr(3); + } else { + itemType = BaseModel.MODEL_TYPE_NOTE; + } - let item = await BaseItem.loadItemByField(itemType, 'title', pattern); - if (!item) return cmdError(this, _('No item with title "%s" found.', pattern), end); - await BaseItem.deleteItem(itemType, item.id); + let item = await BaseItem.loadItemByField(itemType, 'title', pattern); + if (!item) throw new Error(_('No item with title "%s" found.', pattern)); + await BaseItem.deleteItem(itemType, item.id); - if (currentFolder && currentFolder.id == item.id) { - let f = await Folder.defaultFolder(); - switchCurrentFolder(f); - } - } else { // Handle it as a glob pattern - let notes = await Note.previews(currentFolder.id, { titlePattern: pattern }); - if (!notes.length) return cmdError(this, _('No note matches this pattern: "%s"', pattern), end); - let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length)); - if (ok) { - for (let i = 0; i < notes.length; i++) { - await Note.delete(notes[i].id); + if (currentFolder && currentFolder.id == item.id) { + let f = await Folder.defaultFolder(); + switchCurrentFolder(f); + } + } else { // Handle it as a glob pattern + let notes = await Note.previews(currentFolder.id, { titlePattern: pattern }); + if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern)); + let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length)); + if (ok) { + for (let i = 0; i < notes.length; i++) { + await Note.delete(notes[i].id); + } } } + } catch (error) { + this.log(error); } end(); @@ -226,15 +231,19 @@ commands.push({ usage: 'mv ', description: 'Moves the notes matching to .', action: async function(args, end) { - let pattern = args['pattern']; + try { + let pattern = args['pattern']; - let folder = await Folder.loadByField('title', args['notebook']); - if (!folder) return cmdError(this, _('No folder with title "%s"', args['notebook']), end); - let notes = await Note.previews(currentFolder.id, { titlePattern: pattern }); - if (!notes.length) return cmdError(this, _('No note matches this pattern: "%s"', pattern), end); + let folder = await Folder.loadByField('title', args['notebook']); + if (!folder) throw new Error(_('No folder with title "%s"', args['notebook'])); + let notes = await Note.previews(currentFolder.id, { titlePattern: pattern }); + if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern)); - for (let i = 0; i < notes.length; i++) { - await Note.save({ id: notes[i].id, parent_id: folder.id }); + for (let i = 0; i < notes.length; i++) { + await Note.save({ id: notes[i].id, parent_id: folder.id }); + } + } catch (error) { + this.log(error); } end(); @@ -252,41 +261,45 @@ commands.push({ ['-t, --type ', 'Displays only the items of the specific type(s). Can be `n` for notes, `t` for todos, or `nt` for notes and todos (eg. `-tt` would display only the todos, while `-ttd` would display notes and todos.'], ], action: async function(args, end) { - let pattern = args['pattern']; - let suffix = ''; - let items = []; - let options = args.options; + try { + let pattern = args['pattern']; + let suffix = ''; + let items = []; + let options = args.options; - let queryOptions = {}; - if (options.lines) queryOptions.limit = options.lines; - if (options.sort) { - queryOptions.orderBy = options.sort; - queryOptions.orderByDir = 'ASC'; - } - if (options.reverse === true) queryOptions.orderByDir = queryOptions.orderByDir == 'ASC' ? 'DESC' : 'ASC'; - queryOptions.caseInsensitive = true; - if (options.type) { - queryOptions.itemTypes = []; - if (options.type.indexOf('n') >= 0) queryOptions.itemTypes.push('note'); - if (options.type.indexOf('t') >= 0) queryOptions.itemTypes.push('todo'); - } - if (pattern) queryOptions.titlePattern = pattern; - - if (pattern == '..') { - items = await Folder.all(queryOptions); - suffix = '/'; - } else { - items = await Note.previews(currentFolder.id, queryOptions); - } - - for (let i = 0; i < items.length; i++) { - let item = items[i]; - let line = ''; - if (!!item.is_todo) { - line += sprintf('[%s] ', !!item.todo_completed ? 'X' : ' '); + let queryOptions = {}; + if (options.lines) queryOptions.limit = options.lines; + if (options.sort) { + queryOptions.orderBy = options.sort; + queryOptions.orderByDir = 'ASC'; } - line += item.title + suffix; - this.log(line); + if (options.reverse === true) queryOptions.orderByDir = queryOptions.orderByDir == 'ASC' ? 'DESC' : 'ASC'; + queryOptions.caseInsensitive = true; + if (options.type) { + queryOptions.itemTypes = []; + if (options.type.indexOf('n') >= 0) queryOptions.itemTypes.push('note'); + if (options.type.indexOf('t') >= 0) queryOptions.itemTypes.push('todo'); + } + if (pattern) queryOptions.titlePattern = pattern; + + if (pattern == '..') { + items = await Folder.all(queryOptions); + suffix = '/'; + } else { + items = await Note.previews(currentFolder.id, queryOptions); + } + + for (let i = 0; i < items.length; i++) { + let item = items[i]; + let line = ''; + if (!!item.is_todo) { + line += sprintf('[%s] ', !!item.todo_completed ? 'X' : ' '); + } + line += item.title + suffix; + this.log(line); + } + } catch (Error) { + this.log(error); } end(); @@ -315,59 +328,64 @@ commands.push({ ['--fuzzy-matching', 'For debugging purposes. Do not use.'], ], action: async function(args, end) { - let filePath = args.file; - let folder = null; - let folderTitle = args['notebook']; + try { + let filePath = args.file; + let folder = null; + let folderTitle = args['notebook']; - if (folderTitle) { - folder = await Folder.loadByField('title', folderTitle); - if (!folder) return cmdError(this, _('Folder does not exists: "%s"', folderTitle), end); - } else { - folderTitle = filename(filePath); - folderTitle = _('Imported - %s', folderTitle); - let inc = 0; - while (true) { - let t = folderTitle + (inc ? ' (' + inc + ')' : ''); - let f = await Folder.loadByField('title', t); - if (!f) { - folderTitle = t; - break; + if (folderTitle) { + folder = await Folder.loadByField('title', folderTitle); + if (!folder) return cmdError(this, _('Folder does not exists: "%s"', folderTitle), end); + } else { + folderTitle = filename(filePath); + folderTitle = _('Imported - %s', folderTitle); + let inc = 0; + while (true) { + let t = folderTitle + (inc ? ' (' + inc + ')' : ''); + let f = await Folder.loadByField('title', t); + if (!f) { + folderTitle = t; + break; + } + inc++; } - inc++; } + + let ok = await cmdPromptConfirm(this, _('File "%s" will be imported into notebook "%s". Continue?', basename(filePath), folderTitle)) + + if (!ok) { + end(); + return; + } + + let redrawnCalled = false; + let options = { + fuzzyMatching: args.options['fuzzy-matching'] === true, + onProgress: (progressState) => { + let line = []; + line.push(_('Found: %d.', progressState.loaded)); + line.push(_('Created: %d.', progressState.created)); + if (progressState.updated) line.push(_('Updated: %d.', progressState.updated)); + if (progressState.skipped) line.push(_('Skipped: %d.', progressState.skipped)); + if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated)); + redrawnCalled = true; + vorpal.ui.redraw(line.join(' ')); + }, + onError: (error) => { + let s = error.trace ? error.trace : error.toString(); + this.log(s); + }, + } + + folder = !folder ? await Folder.save({ title: folderTitle }) : folder; + this.log(_('Importing notes...')); + await importEnex(folder.id, filePath, options); + if (redrawnCalled) vorpal.ui.redraw.done(); + this.log(_('Done.')); + } catch (error) { + this.log(error); } - let ok = await cmdPromptConfirm(this, _('File "%s" will be imported into notebook "%s". Continue?', basename(filePath), folderTitle)) - - if (!ok) { - end(); - return; - } - - let redrawnCalled = false; - let options = { - fuzzyMatching: args.options['fuzzy-matching'] === true, - onProgress: (progressState) => { - let line = []; - line.push(_('Found: %d.', progressState.loaded)); - line.push(_('Created: %d.', progressState.created)); - if (progressState.updated) line.push(_('Updated: %d.', progressState.updated)); - if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated)); - redrawnCalled = true; - vorpal.ui.redraw(line.join(' ')); - }, - onError: (error) => { - let s = error.trace ? error.trace : error.toString(); - this.log(s); - }, - } - - folder = !folder ? await Folder.save({ title: folderTitle }) : folder; - this.log(_('Importing notes...')); - await importEnex(folder.id, filePath, options); - if (redrawnCalled) vorpal.ui.redraw.done(); - this.log(_('Done.')); - end(); }, }); diff --git a/CliClient/package.json b/CliClient/package.json index 2a0633383c..2fefa5e541 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -1,6 +1,6 @@ { "name": "joplin-cli", - "version": "0.8.9", + "version": "0.8.11", "bin": { "joplin": "./main.sh" }, diff --git a/lib/base-model.js b/lib/base-model.js index cebb403fa9..3b05005396 100644 --- a/lib/base-model.js +++ b/lib/base-model.js @@ -271,7 +271,17 @@ class BaseModel { } static filter(model) { - return model; + if (!model) return model; + + let output = Object.assign({}, model); + for (let n in output) { + if (!output.hasOwnProperty(n)) continue; + // The SQLite database doesn't have booleans so cast everything to int + if (output[n] === true) output[n] = 1; + if (output[n] === false) output[n] = 0; + } + + return output; } static delete(id, options = null) { diff --git a/lib/database.js b/lib/database.js index 240d614fa7..667d41dc51 100644 --- a/lib/database.js +++ b/lib/database.js @@ -1,7 +1,9 @@ import { uuid } from 'lib/uuid.js'; import { promiseChain } from 'lib/promise-utils.js'; import { Logger } from 'lib/logger.js' +import { time } from 'lib/time-utils.js' import { _ } from 'lib/locale.js' +import { sprintf } from 'sprintf-js'; const structureSql = ` CREATE TABLE folders ( @@ -193,11 +195,29 @@ class Database { }); } - exec(sql, params = null) { - this.logQuery(sql, params); - return this.driver().exec(sql, params).catch((error) => { - throw this.sqliteErrorToJsError(error, sql, params); - }); + async exec(sql, params = null) { + let result = null; + let waitTime = 50; + let totalWaitTime = 0; + while (true) { + try { + this.logQuery(sql, params); + let result = await this.driver().exec(sql, params); + return result;; // No exception was thrown + } catch (error) { + throw error; + if (error && error.code == 'SQLITE_IOERR') { + if (totalWaitTime >= 20000) throw error; + this.logger().warn(sprintf('SQLITE_IOERR: will retry in %s milliseconds', waitTime)); + this.logger().warn('Error was: ' + error.toString()); + await time.msleep(waitTime); + totalWaitTime += waitTime; + waitTime *= 1.5; + } else { + throw this.sqliteErrorToJsError(error, sql, params); + } + } + } } transactionExecBatch(queries) { diff --git a/lib/models/note.js b/lib/models/note.js index 21cf505c37..124a792f38 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -105,7 +105,7 @@ class Note extends BaseItem { static filter(note) { if (!note) return note; - let output = Object.assign({}, note); + let output = super.filter(note); if ('longitude' in output) output.longitude = Number(!output.longitude ? 0 : output.longitude).toFixed(8); if ('latitude' in output) output.latitude = Number(!output.latitude ? 0 : output.latitude).toFixed(8); if ('altitude' in output) output.altitude = Number(!output.altitude ? 0 : output.altitude).toFixed(4); diff --git a/lib/onedrive-api.js b/lib/onedrive-api.js index 56a71608ed..8a6e55b6e9 100644 --- a/lib/onedrive-api.js +++ b/lib/onedrive-api.js @@ -224,7 +224,8 @@ class OneDriveApi { enableServerDestroy(server); - console.info('Please open this URL in your browser to authentify the application: ' + authCodeUrl); + console.info('Please open this URL in your browser to authentify the application:'); + console.info(authCodeUrl); }); } diff --git a/lib/time-utils.js b/lib/time-utils.js index 9ff6911a54..837ef2bbc1 100644 --- a/lib/time-utils.js +++ b/lib/time-utils.js @@ -18,6 +18,18 @@ let time = { return moment.unix(ms / 1000).utc().format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z'; }, + msleep(ms) { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve(); + }, ms); + }); + }, + + sleep(seconds) { + return this.msleep(seconds * 1000); + }, + } export { time }; \ No newline at end of file