Improved enex import

CliClient/app/import-enex.js

@@ -13,25 +13,6 @@ const Promise = require('promise');
 const fs = require('fs-extra');
 const stringToStream = require('string-to-stream')
 
-let existingTimestamps = [];
-
-function uniqueCreatedTimestamp(timestamp) {
-	if (existingTimestamps.indexOf(timestamp) < 0) {
-		existingTimestamps.push(timestamp);
-		return timestamp;
-	}
-
-	for (let i = 1; i <= 999; i++) {
-		let t = timestamp + i;
-		if (existingTimestamps.indexOf(t) < 0) {
-			existingTimestamps.push(t);
-			return t;
-		}
-	}
-
-	return timestamp;
-}
-
 function dateToTimestamp(s, zeroIfInvalid = false) {
 	let m = moment(s, 'YYYYMMDDTHHmmssZ');
 	if (!m.isValid()) {
@@ -69,7 +50,7 @@ function createNoteId(note) {
 }
 
 async function fuzzyMatch(note) {
-	let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', note.created_time);
+	let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', [note.created_time]);
 	if (!notes.length) return null;
 	if (notes.length === 1) return notes[0];
 
@@ -92,6 +73,7 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
 	let result = {
 		noteCreated: false,
 		noteUpdated: false,
+		noteSkipped: false,
 		resourcesCreated: 0,
 	};
 
@@ -103,14 +85,16 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
 
 		// TODO: also save resources
 
-		if (!Object.getOwnPropertyNames(diff).length) return;
+		if (!Object.getOwnPropertyNames(diff).length) {
+			result.noteSkipped = true;
+			// TODO: also save resources
+			return result;
+		}
 
 		diff.id = existingNote.id;
 		diff.type_ = existingNote.type_;
-		return Note.save(diff, { autoTimestamp: false }).then(() => {
+		await Note.save(diff, { autoTimestamp: false })
 		result.noteUpdated = true;
-			return result;
-		});
 	} else {
 		for (let i = 0; i < note.resources.length; i++) {
 			let resource = note.resources[i];
@@ -128,14 +112,14 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
 			result.resourcesCreated++;
 		}
 
-		return Note.save(note, {
+		await Note.save(note, {
 			isNew: true,
 			autoTimestamp: false,
-		}).then(() => {
-			result.noteCreated = true;
-			return result;
 		});
+		result.noteCreated = true;
 	}
+
+	return result;
 }
 
 function importEnex(parentFolderId, filePath, importOptions = null) {
@@ -144,11 +128,35 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 	if (!('onProgress' in importOptions)) importOptions.onProgress = function(state) {};
 	if (!('onError' in importOptions)) importOptions.onError = function(error) {};
 
+	// Some notes were created with the exact same timestamp, for example when they were
+	// batch imported. In order to make fuzzy matching easier, this function ensures
+	// that each timestamp is unique.
+	let existingTimestamps = [];
+	function uniqueCreatedTimestamp(timestamp) {
+		return timestamp;
+
+		if (existingTimestamps.indexOf(timestamp) < 0) {
+			existingTimestamps.push(timestamp);
+			return timestamp;
+		}
+
+		for (let i = 1; i <= 999; i++) {
+			let t = timestamp + i;
+			if (existingTimestamps.indexOf(t) < 0) {
+				existingTimestamps.push(t);
+				return t;
+			}
+		}
+
+		return timestamp;
+	}
+
 	return new Promise((resolve, reject) => {
 		let progressState = {
 			loaded: 0,
 			created: 0,
 			updated: 0,
+			skipped: 0,
 			resourcesCreated: 0,
 		};
 
@@ -182,7 +190,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 		}
 
 		async function processNotes() {
-			if (processingNotes) return;
+			if (processingNotes) return false;
+
 			processingNotes = true;
 			stream.pause();
 
@@ -204,6 +213,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 							progressState.updated++;
 						} else if (result.noteCreated) {
 							progressState.created++;
+						} else if (result.noteSkipped) {
+							progressState.skipped++;
 						}
 						progressState.resourcesCreated += result.resourcesCreated;
 						importOptions.onProgress(progressState);
@@ -214,6 +225,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 			return promiseChain(chain).then(() => {
 				stream.resume();
 				processingNotes = false;
+				return true;
 			});
 		}
 
@@ -354,12 +366,12 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 		saxStream.on('end', function() {
 			// Wait till there is no more notes to process.
 			let iid = setInterval(() => {
-				if (notes.length) {
-					processNotes();
-				} else {
-					clearInterval(iid);
+				processNotes().then((allDone) => {
+					if (allDone) {
+						clearTimeout(iid);
 						resolve();
 					}
+				});
 			}, 500);
 		});
 

CliClient/app/main.js

@@ -63,10 +63,11 @@ commands.push({
 	aliases: ['mkdir'],
 	description: 'Creates a new notebook',
 	action: function(args, end) {
-		Folder.save({ title: args['notebook'] }).catch((error) => {
-			this.log(error);
-		}).then((folder) => {
+		Folder.save({ title: args['notebook'] }).then((folder) => {
 			switchCurrentFolder(folder);
+		}).catch((error) => {
+			this.log(error);
+		}).then(() => {
 			end();
 		});
 	},
@@ -187,6 +188,7 @@ commands.push({
 	usage: 'rm <pattern>',
 	description: 'Deletes the given item. For a notebook, all the notes within that notebook will be deleted. Use `rm ../<notebook>` to delete a notebook.',
 	action: async function(args, end) {
+		try {
 			let pattern = args['pattern'];
 			let itemType = null;
 
@@ -199,7 +201,7 @@ commands.push({
 				}
 
 				let item = await BaseItem.loadItemByField(itemType, 'title', pattern);
-			if (!item) return cmdError(this, _('No item with title "%s" found.', pattern), end);
+				if (!item) throw new Error(_('No item with title "%s" found.', pattern));
 				await BaseItem.deleteItem(itemType, item.id);
 
 				if (currentFolder && currentFolder.id == item.id) {
@@ -208,7 +210,7 @@ commands.push({
 				}
 			} else { // Handle it as a glob pattern
 				let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
-			if (!notes.length) return cmdError(this, _('No note matches this pattern: "%s"', pattern), end);
+				if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern));
 				let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length));
 				if (ok) {
 					for (let i = 0; i < notes.length; i++) {
@@ -216,6 +218,9 @@ commands.push({
 					}
 				}
 			}
+		} catch (error) {
+			this.log(error);
+		}
 
 		end();
 	},
@@ -226,16 +231,20 @@ commands.push({
 	usage: 'mv <pattern> <notebook>',
 	description: 'Moves the notes matching <pattern> to <notebook>.',
 	action: async function(args, end) {
+		try {
 			let pattern = args['pattern'];
 
 			let folder = await Folder.loadByField('title', args['notebook']);
-		if (!folder) return cmdError(this, _('No folder with title "%s"', args['notebook']), end);
+			if (!folder) throw new Error(_('No folder with title "%s"', args['notebook']));
 			let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
-		if (!notes.length) return cmdError(this, _('No note matches this pattern: "%s"', pattern), end);
+			if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern));
 
 			for (let i = 0; i < notes.length; i++) {
 				await Note.save({ id: notes[i].id, parent_id: folder.id });
 			}
+		} catch (error) {
+			this.log(error);
+		}
 
 		end();
 	},
@@ -252,6 +261,7 @@ commands.push({
 		['-t, --type <type>', 'Displays only the items of the specific type(s). Can be `n` for notes, `t` for todos, or `nt` for notes and todos (eg. `-tt` would display only the todos, while `-ttd` would display notes and todos.'],
 	],
 	action: async function(args, end) {
+		try {
 			let pattern = args['pattern'];
 			let suffix = '';
 			let items = [];
@@ -288,6 +298,9 @@ commands.push({
 				line += item.title + suffix;
 				this.log(line);
 			}
+		} catch (Error) {
+			this.log(error);
+		}
 
 		end();
 	},
@@ -315,6 +328,7 @@ commands.push({
 		['--fuzzy-matching', 'For debugging purposes. Do not use.'],
 	],
 	action: async function(args, end) {
+		try {
 			let filePath = args.file;
 			let folder = null;
 			let folderTitle = args['notebook'];
@@ -352,6 +366,7 @@ commands.push({
 					line.push(_('Found: %d.', progressState.loaded));
 					line.push(_('Created: %d.', progressState.created));
 					if (progressState.updated) line.push(_('Updated: %d.', progressState.updated));
+					if (progressState.skipped) line.push(_('Skipped: %d.', progressState.skipped));
 					if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated));
 					redrawnCalled = true;
 					vorpal.ui.redraw(line.join(' '));
@@ -367,6 +382,9 @@ commands.push({
 			await importEnex(folder.id, filePath, options);
 			if (redrawnCalled) vorpal.ui.redraw.done();
 			this.log(_('Done.'));
+		} catch (error) {
+			this.log(error);
+		}
 
 		end();			
 	},

CliClient/package.json

@@ -1,6 +1,6 @@
 {
   "name": "joplin-cli",
-  "version": "0.8.9",
+  "version": "0.8.11",
   "bin": {
     "joplin": "./main.sh"
   },

lib/base-model.js

@@ -271,7 +271,17 @@ class BaseModel {
 	}
 
 	static filter(model) {
-		return model;
+		if (!model) return model;
+
+		let output = Object.assign({}, model);
+		for (let n in output) {
+			if (!output.hasOwnProperty(n)) continue;
+			// The SQLite database doesn't have booleans so cast everything to int
+			if (output[n] === true) output[n] = 1;
+			if (output[n] === false) output[n] = 0;
+		}
+		
+		return output;
 	}
 
 	static delete(id, options = null) {

lib/database.js

@@ -1,7 +1,9 @@
 import { uuid } from 'lib/uuid.js';
 import { promiseChain } from 'lib/promise-utils.js';
 import { Logger } from 'lib/logger.js'
+import { time } from 'lib/time-utils.js'
 import { _ } from 'lib/locale.js'
+import { sprintf } from 'sprintf-js';
 
 const structureSql = `
 CREATE TABLE folders (
@@ -193,11 +195,29 @@ class Database {
 		});
 	}
 
-	exec(sql, params = null) {
+	async exec(sql, params = null) {
+		let result = null;
+		let waitTime = 50;
+		let totalWaitTime = 0;
+		while (true) {
+			try {
 				this.logQuery(sql, params);
-		return this.driver().exec(sql, params).catch((error) => {
+				let result = await this.driver().exec(sql, params);
+				return result;; // No exception was thrown
+			} catch (error) {
+				throw error;
+				if (error && error.code == 'SQLITE_IOERR') {
+					if (totalWaitTime >= 20000) throw error;
+					this.logger().warn(sprintf('SQLITE_IOERR: will retry in %s milliseconds', waitTime));
+					this.logger().warn('Error was: ' + error.toString());
+					await time.msleep(waitTime);
+					totalWaitTime += waitTime;
+					waitTime *= 1.5;
+				} else {
 					throw this.sqliteErrorToJsError(error, sql, params);
-		});
+				}			
+			}
+		}
 	}
 
 	transactionExecBatch(queries) {

lib/models/note.js

@@ -105,7 +105,7 @@ class Note extends BaseItem {
 	static filter(note) {
 		if (!note) return note;
 
-		let output = Object.assign({}, note);
+		let output = super.filter(note);
 		if ('longitude' in output) output.longitude = Number(!output.longitude ? 0 : output.longitude).toFixed(8);
 		if ('latitude' in output) output.latitude = Number(!output.latitude ? 0 : output.latitude).toFixed(8);
 		if ('altitude' in output) output.altitude = Number(!output.altitude ? 0 : output.altitude).toFixed(4);

lib/onedrive-api.js

@@ -224,7 +224,8 @@ class OneDriveApi {
 
 			enableServerDestroy(server);
 
-			console.info('Please open this URL in your browser to authentify the application: ' + authCodeUrl);
+			console.info('Please open this URL in your browser to authentify the application:');
+			console.info(authCodeUrl);
 		});
 	}
 

lib/time-utils.js

@@ -18,6 +18,18 @@ let time = {
 		return moment.unix(ms / 1000).utc().format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z';
 	},
 
+	msleep(ms) {
+		return new Promise((resolve, reject) => {
+			setTimeout(() => {
+				resolve();
+			}, ms);
+		});
+	},
+
+	sleep(seconds) {
+		return this.msleep(seconds * 1000);
+	},
+
 }
 
 export { time };