Improved enex import

CliClient/app/import-enex.js

@@ -13,25 +13,6 @@ const Promise = require('promise');
 const fs = require('fs-extra');
 const stringToStream = require('string-to-stream')
 
-let existingTimestamps = [];
-
-function uniqueCreatedTimestamp(timestamp) {
-	if (existingTimestamps.indexOf(timestamp) < 0) {
-		existingTimestamps.push(timestamp);
-		return timestamp;
-	}
-
-	for (let i = 1; i <= 999; i++) {
-		let t = timestamp + i;
-		if (existingTimestamps.indexOf(t) < 0) {
-			existingTimestamps.push(t);
-			return t;
-		}
-	}
-
-	return timestamp;
-}
-
 function dateToTimestamp(s, zeroIfInvalid = false) {
 	let m = moment(s, 'YYYYMMDDTHHmmssZ');
 	if (!m.isValid()) {
@@ -69,7 +50,7 @@ function createNoteId(note) {
 }
 
 async function fuzzyMatch(note) {
-	let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', note.created_time);
+	let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', [note.created_time]);
 	if (!notes.length) return null;
 	if (notes.length === 1) return notes[0];
 
@@ -92,6 +73,7 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
 	let result = {
 		noteCreated: false,
 		noteUpdated: false,
+		noteSkipped: false,
 		resourcesCreated: 0,
 	};
 
@@ -103,14 +85,16 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
 
 		// TODO: also save resources
 
-		if (!Object.getOwnPropertyNames(diff).length) return;
+		if (!Object.getOwnPropertyNames(diff).length) {
+			result.noteSkipped = true;
+			// TODO: also save resources
+			return result;
+		}
 
 		diff.id = existingNote.id;
 		diff.type_ = existingNote.type_;
-		return Note.save(diff, { autoTimestamp: false }).then(() => {
+		await Note.save(diff, { autoTimestamp: false })
-			result.noteUpdated = true;
+		result.noteUpdated = true;
-			return result;
-		});
 	} else {
 		for (let i = 0; i < note.resources.length; i++) {
 			let resource = note.resources[i];
@@ -128,14 +112,14 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
 			result.resourcesCreated++;
 		}
 
-		return Note.save(note, {
+		await Note.save(note, {
 			isNew: true,
 			autoTimestamp: false,
-		}).then(() => {
-			result.noteCreated = true;
-			return result;
 		});
+		result.noteCreated = true;
 	}
+
+	return result;
 }
 
 function importEnex(parentFolderId, filePath, importOptions = null) {
@@ -144,11 +128,35 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 	if (!('onProgress' in importOptions)) importOptions.onProgress = function(state) {};
 	if (!('onError' in importOptions)) importOptions.onError = function(error) {};
 
+	// Some notes were created with the exact same timestamp, for example when they were
+	// batch imported. In order to make fuzzy matching easier, this function ensures
+	// that each timestamp is unique.
+	let existingTimestamps = [];
+	function uniqueCreatedTimestamp(timestamp) {
+		return timestamp;
+
+		if (existingTimestamps.indexOf(timestamp) < 0) {
+			existingTimestamps.push(timestamp);
+			return timestamp;
+		}
+
+		for (let i = 1; i <= 999; i++) {
+			let t = timestamp + i;
+			if (existingTimestamps.indexOf(t) < 0) {
+				existingTimestamps.push(t);
+				return t;
+			}
+		}
+
+		return timestamp;
+	}
+
 	return new Promise((resolve, reject) => {
 		let progressState = {
 			loaded: 0,
 			created: 0,
 			updated: 0,
+			skipped: 0,
 			resourcesCreated: 0,
 		};
 
@@ -182,7 +190,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 		}
 
 		async function processNotes() {
-			if (processingNotes) return;
+			if (processingNotes) return false;
+
 			processingNotes = true;
 			stream.pause();
 
@@ -204,6 +213,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 							progressState.updated++;
 						} else if (result.noteCreated) {
 							progressState.created++;
+						} else if (result.noteSkipped) {
+							progressState.skipped++;
 						}
 						progressState.resourcesCreated += result.resourcesCreated;
 						importOptions.onProgress(progressState);
@@ -214,6 +225,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 			return promiseChain(chain).then(() => {
 				stream.resume();
 				processingNotes = false;
+				return true;
 			});
 		}
 
@@ -354,12 +366,12 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 		saxStream.on('end', function() {
 			// Wait till there is no more notes to process.
 			let iid = setInterval(() => {
-				if (notes.length) {
+				processNotes().then((allDone) => {
-					processNotes();
+					if (allDone) {
-				} else {
+						clearTimeout(iid);
-					clearInterval(iid);
+						resolve();
-					resolve();
+					}
-				}
+				});
 			}, 500);
 		});
 

CliClient/app/main.js

@@ -63,10 +63,11 @@ commands.push({
 	aliases: ['mkdir'],
 	description: 'Creates a new notebook',
 	action: function(args, end) {
-		Folder.save({ title: args['notebook'] }).catch((error) => {
+		Folder.save({ title: args['notebook'] }).then((folder) => {
-			this.log(error);
-		}).then((folder) => {
 			switchCurrentFolder(folder);
+		}).catch((error) => {
+			this.log(error);
+		}).then(() => {
 			end();
 		});
 	},
@@ -187,34 +188,38 @@ commands.push({
 	usage: 'rm <pattern>',
 	description: 'Deletes the given item. For a notebook, all the notes within that notebook will be deleted. Use `rm ../<notebook>` to delete a notebook.',
 	action: async function(args, end) {
-		let pattern = args['pattern'];
+		try {
-		let itemType = null;
+			let pattern = args['pattern'];
+			let itemType = null;
 
-		if (pattern.indexOf('*') < 0) { // Handle it as a simple title
+			if (pattern.indexOf('*') < 0) { // Handle it as a simple title
-			if (pattern.substr(0, 3) == '../') {
+				if (pattern.substr(0, 3) == '../') {
-				itemType = BaseModel.MODEL_TYPE_FOLDER;
+					itemType = BaseModel.MODEL_TYPE_FOLDER;
-				pattern = pattern.substr(3);
+					pattern = pattern.substr(3);
-			} else {
+				} else {
-				itemType = BaseModel.MODEL_TYPE_NOTE;
+					itemType = BaseModel.MODEL_TYPE_NOTE;
-			}
+				}
 
-			let item = await BaseItem.loadItemByField(itemType, 'title', pattern);
+				let item = await BaseItem.loadItemByField(itemType, 'title', pattern);
-			if (!item) return cmdError(this, _('No item with title "%s" found.', pattern), end);
+				if (!item) throw new Error(_('No item with title "%s" found.', pattern));
-			await BaseItem.deleteItem(itemType, item.id);
+				await BaseItem.deleteItem(itemType, item.id);
 
-			if (currentFolder && currentFolder.id == item.id) {
+				if (currentFolder && currentFolder.id == item.id) {
-				let f = await Folder.defaultFolder();
+					let f = await Folder.defaultFolder();
-				switchCurrentFolder(f);
+					switchCurrentFolder(f);
-			}
+				}
-		} else { // Handle it as a glob pattern
+			} else { // Handle it as a glob pattern
-			let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
+				let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
-			if (!notes.length) return cmdError(this, _('No note matches this pattern: "%s"', pattern), end);
+				if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern));
-			let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length));
+				let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length));
-			if (ok) {
+				if (ok) {
-				for (let i = 0; i < notes.length; i++) {
+					for (let i = 0; i < notes.length; i++) {
-					await Note.delete(notes[i].id);
+						await Note.delete(notes[i].id);
+					}
 				}
 			}
+		} catch (error) {
+			this.log(error);
 		}
 
 		end();
@@ -226,15 +231,19 @@ commands.push({
 	usage: 'mv <pattern> <notebook>',
 	description: 'Moves the notes matching <pattern> to <notebook>.',
 	action: async function(args, end) {
-		let pattern = args['pattern'];
+		try {
+			let pattern = args['pattern'];
 
-		let folder = await Folder.loadByField('title', args['notebook']);
+			let folder = await Folder.loadByField('title', args['notebook']);
-		if (!folder) return cmdError(this, _('No folder with title "%s"', args['notebook']), end);
+			if (!folder) throw new Error(_('No folder with title "%s"', args['notebook']));
-		let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
+			let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
-		if (!notes.length) return cmdError(this, _('No note matches this pattern: "%s"', pattern), end);
+			if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern));
 
-		for (let i = 0; i < notes.length; i++) {
+			for (let i = 0; i < notes.length; i++) {
-			await Note.save({ id: notes[i].id, parent_id: folder.id });
+				await Note.save({ id: notes[i].id, parent_id: folder.id });
+			}
+		} catch (error) {
+			this.log(error);
 		}
 
 		end();
@@ -252,41 +261,45 @@ commands.push({
 		['-t, --type <type>', 'Displays only the items of the specific type(s). Can be `n` for notes, `t` for todos, or `nt` for notes and todos (eg. `-tt` would display only the todos, while `-ttd` would display notes and todos.'],
 	],
 	action: async function(args, end) {
-		let pattern = args['pattern'];
+		try {
-		let suffix = '';
+			let pattern = args['pattern'];
-		let items = [];
+			let suffix = '';
-		let options = args.options;
+			let items = [];
+			let options = args.options;
 
-		let queryOptions = {};
+			let queryOptions = {};
-		if (options.lines) queryOptions.limit = options.lines;
+			if (options.lines) queryOptions.limit = options.lines;
-		if (options.sort) {
+			if (options.sort) {
-			queryOptions.orderBy = options.sort;
+				queryOptions.orderBy = options.sort;
-			queryOptions.orderByDir = 'ASC';
+				queryOptions.orderByDir = 'ASC';
-		}
-		if (options.reverse === true) queryOptions.orderByDir = queryOptions.orderByDir == 'ASC' ? 'DESC' : 'ASC';
-		queryOptions.caseInsensitive = true;
-		if (options.type) {
-			queryOptions.itemTypes = [];
-			if (options.type.indexOf('n') >= 0) queryOptions.itemTypes.push('note');
-			if (options.type.indexOf('t') >= 0) queryOptions.itemTypes.push('todo');
-		}
-		if (pattern) queryOptions.titlePattern = pattern;
-
-		if (pattern == '..') {
-			items = await Folder.all(queryOptions);
-			suffix = '/';
-		} else {
-			items = await Note.previews(currentFolder.id, queryOptions);
-		}
-
-		for (let i = 0; i < items.length; i++) {
-			let item = items[i];
-			let line = '';
-			if (!!item.is_todo) {
-				line += sprintf('[%s] ', !!item.todo_completed ? 'X' : ' ');
 			}
-			line += item.title + suffix;
+			if (options.reverse === true) queryOptions.orderByDir = queryOptions.orderByDir == 'ASC' ? 'DESC' : 'ASC';
-			this.log(line);
+			queryOptions.caseInsensitive = true;
+			if (options.type) {
+				queryOptions.itemTypes = [];
+				if (options.type.indexOf('n') >= 0) queryOptions.itemTypes.push('note');
+				if (options.type.indexOf('t') >= 0) queryOptions.itemTypes.push('todo');
+			}
+			if (pattern) queryOptions.titlePattern = pattern;
+
+			if (pattern == '..') {
+				items = await Folder.all(queryOptions);
+				suffix = '/';
+			} else {
+				items = await Note.previews(currentFolder.id, queryOptions);
+			}
+
+			for (let i = 0; i < items.length; i++) {
+				let item = items[i];
+				let line = '';
+				if (!!item.is_todo) {
+					line += sprintf('[%s] ', !!item.todo_completed ? 'X' : ' ');
+				}
+				line += item.title + suffix;
+				this.log(line);
+			}
+		} catch (Error) {
+			this.log(error);
 		}
 
 		end();
@@ -315,59 +328,64 @@ commands.push({
 		['--fuzzy-matching', 'For debugging purposes. Do not use.'],
 	],
 	action: async function(args, end) {
-		let filePath = args.file;
+		try {
-		let folder = null;
+			let filePath = args.file;
-		let folderTitle = args['notebook'];
+			let folder = null;
+			let folderTitle = args['notebook'];
 
-		if (folderTitle) {
+			if (folderTitle) {
-			folder = await Folder.loadByField('title', folderTitle);
+				folder = await Folder.loadByField('title', folderTitle);
-			if (!folder) return cmdError(this, _('Folder does not exists: "%s"', folderTitle), end);
+				if (!folder) return cmdError(this, _('Folder does not exists: "%s"', folderTitle), end);
-		} else {
+			} else {
-			folderTitle = filename(filePath);
+				folderTitle = filename(filePath);
-			folderTitle = _('Imported - %s', folderTitle);
+				folderTitle = _('Imported - %s', folderTitle);
-			let inc = 0;
+				let inc = 0;
-			while (true) {
+				while (true) {
-				let t = folderTitle + (inc ? ' (' + inc + ')' : '');
+					let t = folderTitle + (inc ? ' (' + inc + ')' : '');
-				let f = await Folder.loadByField('title', t);
+					let f = await Folder.loadByField('title', t);
-				if (!f) {
+					if (!f) {
-					folderTitle = t;
+						folderTitle = t;
-					break;
+						break;
+					}
+					inc++;
 				}
-				inc++;
 			}
+
+			let ok = await cmdPromptConfirm(this, _('File "%s" will be imported into notebook "%s". Continue?', basename(filePath), folderTitle))
+
+			if (!ok) {
+				end();
+				return;
+			}
+
+			let redrawnCalled = false;
+			let options = {
+				fuzzyMatching: args.options['fuzzy-matching'] === true,
+				onProgress: (progressState) => {
+					let line = [];
+					line.push(_('Found: %d.', progressState.loaded));
+					line.push(_('Created: %d.', progressState.created));
+					if (progressState.updated) line.push(_('Updated: %d.', progressState.updated));
+					if (progressState.skipped) line.push(_('Skipped: %d.', progressState.skipped));
+					if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated));
+					redrawnCalled = true;
+					vorpal.ui.redraw(line.join(' '));
+				},
+				onError: (error) => {
+					let s = error.trace ? error.trace : error.toString();
+					this.log(s);
+				},
+			}
+
+			folder = !folder ? await Folder.save({ title: folderTitle }) : folder;
+			this.log(_('Importing notes...'));
+			await importEnex(folder.id, filePath, options);
+			if (redrawnCalled) vorpal.ui.redraw.done();
+			this.log(_('Done.'));
+		} catch (error) {
+			this.log(error);
 		}
 
-		let ok = await cmdPromptConfirm(this, _('File "%s" will be imported into notebook "%s". Continue?', basename(filePath), folderTitle))
-
-		if (!ok) {
-			end();
-			return;
-		}
-
-		let redrawnCalled = false;
-		let options = {
-			fuzzyMatching: args.options['fuzzy-matching'] === true,
-			onProgress: (progressState) => {
-				let line = [];
-				line.push(_('Found: %d.', progressState.loaded));
-				line.push(_('Created: %d.', progressState.created));
-				if (progressState.updated) line.push(_('Updated: %d.', progressState.updated));
-				if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated));
-				redrawnCalled = true;
-				vorpal.ui.redraw(line.join(' '));
-			},
-			onError: (error) => {
-				let s = error.trace ? error.trace : error.toString();
-				this.log(s);
-			},
-		}
-
-		folder = !folder ? await Folder.save({ title: folderTitle }) : folder;
-		this.log(_('Importing notes...'));
-		await importEnex(folder.id, filePath, options);
-		if (redrawnCalled) vorpal.ui.redraw.done();
-		this.log(_('Done.'));
-
 		end();			
 	},
 });

CliClient/package.json

@@ -1,6 +1,6 @@
 {
   "name": "joplin-cli",
-  "version": "0.8.9",
+  "version": "0.8.11",
   "bin": {
     "joplin": "./main.sh"
   },

lib/base-model.js

@@ -271,7 +271,17 @@ class BaseModel {
 	}
 
 	static filter(model) {
-		return model;
+		if (!model) return model;
+
+		let output = Object.assign({}, model);
+		for (let n in output) {
+			if (!output.hasOwnProperty(n)) continue;
+			// The SQLite database doesn't have booleans so cast everything to int
+			if (output[n] === true) output[n] = 1;
+			if (output[n] === false) output[n] = 0;
+		}
+		
+		return output;
 	}
 
 	static delete(id, options = null) {

lib/database.js

@@ -1,7 +1,9 @@
 import { uuid } from 'lib/uuid.js';
 import { promiseChain } from 'lib/promise-utils.js';
 import { Logger } from 'lib/logger.js'
+import { time } from 'lib/time-utils.js'
 import { _ } from 'lib/locale.js'
+import { sprintf } from 'sprintf-js';
 
 const structureSql = `
 CREATE TABLE folders (
@@ -193,11 +195,29 @@ class Database {
 		});
 	}
 
-	exec(sql, params = null) {
+	async exec(sql, params = null) {
-		this.logQuery(sql, params);
+		let result = null;
-		return this.driver().exec(sql, params).catch((error) => {
+		let waitTime = 50;
-			throw this.sqliteErrorToJsError(error, sql, params);
+		let totalWaitTime = 0;
-		});
+		while (true) {
+			try {
+				this.logQuery(sql, params);
+				let result = await this.driver().exec(sql, params);
+				return result;; // No exception was thrown
+			} catch (error) {
+				throw error;
+				if (error && error.code == 'SQLITE_IOERR') {
+					if (totalWaitTime >= 20000) throw error;
+					this.logger().warn(sprintf('SQLITE_IOERR: will retry in %s milliseconds', waitTime));
+					this.logger().warn('Error was: ' + error.toString());
+					await time.msleep(waitTime);
+					totalWaitTime += waitTime;
+					waitTime *= 1.5;
+				} else {
+					throw this.sqliteErrorToJsError(error, sql, params);
+				}			
+			}
+		}
 	}
 
 	transactionExecBatch(queries) {

lib/models/note.js

@@ -105,7 +105,7 @@ class Note extends BaseItem {
 	static filter(note) {
 		if (!note) return note;
 
-		let output = Object.assign({}, note);
+		let output = super.filter(note);
 		if ('longitude' in output) output.longitude = Number(!output.longitude ? 0 : output.longitude).toFixed(8);
 		if ('latitude' in output) output.latitude = Number(!output.latitude ? 0 : output.latitude).toFixed(8);
 		if ('altitude' in output) output.altitude = Number(!output.altitude ? 0 : output.altitude).toFixed(4);

lib/onedrive-api.js

@@ -224,7 +224,8 @@ class OneDriveApi {
 
 			enableServerDestroy(server);
 
-			console.info('Please open this URL in your browser to authentify the application: ' + authCodeUrl);
+			console.info('Please open this URL in your browser to authentify the application:');
+			console.info(authCodeUrl);
 		});
 	}
 

lib/time-utils.js

@@ -18,6 +18,18 @@ let time = {
 		return moment.unix(ms / 1000).utc().format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z';
 	},
 
+	msleep(ms) {
+		return new Promise((resolve, reject) => {
+			setTimeout(() => {
+				resolve();
+			}, ms);
+		});
+	},
+
+	sleep(seconds) {
+		return this.msleep(seconds * 1000);
+	},
+
 }
 
 export { time };