Fixed onedrive sync issue

CliClient/app/import-enex.js

@@ -50,19 +50,8 @@ function createNoteId(note) {
 }
 
 async function fuzzyMatch(note) {
-	let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', [note.created_time]);
+	let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ? AND title = ?', [note.created_time, note.title]);
-	if (!notes.length) return null;
+	return notes.length !== 1 ? null : notes[0];
-	if (notes.length === 1) return notes[0];
-
-	for (let i = 0; i < notes.length; i++) {
-		if (notes[i].title == note.title && note.title.trim() != '') return notes[i];
-	}
-
-	for (let i = 0; i < notes.length; i++) {
-		if (notes[i].body == note.body && note.body.trim() != '') return notes[i];
-	}
-
-	return null;
 }
 
 async function saveNoteResources(note) {

CliClient/app/main.js

@@ -42,21 +42,6 @@ let logger = new Logger();
 let dbLogger = new Logger();
 let syncLogger = new Logger();
 
-// commands.push({
-// 	usage: 'root',
-// 	options: [
-// 		['--profile <filePath>', 'Sets the profile path directory.'],
-// 	],
-// 	action: function(args, end) {
-// 		if (args.profile) {
-// 			initArgs.profileDir = args.profile;
-// 			args.splice(0, 2);
-// 		}
-
-// 		end(args);
-// 	},
-// });
-
 commands.push({
 	usage: 'version',
 	description: 'Displays version information',
@@ -433,7 +418,7 @@ function commandByName(name) {
 	for (let i = 0; i < commands.length; i++) {
 		let c = commands[i];
 		let n = c.usage.split(' ');
-		n = n[0];
+		n = n[0].trim();
 		if (n == name) return c;
 		if (c.aliases && c.aliases.indexOf(name) >= 0) return c;
 	}
@@ -453,6 +438,58 @@ function execCommand(name, args) {
 	});
 }
 
+// async function execCommand(args) {
+// 	var parseArgs = require('minimist');
+
+// 	let results = parseArgs(args);
+// 	//var results = vorpal.parse(args, { use: 'minimist' });
+// 	if (!results['_'].length) throw new Error(_('Invalid command: %s', args));
+
+// 	console.info(results);
+
+// 	let commandName = results['_'].splice(0, 1);
+// 	let cmd = commandByName(commandName);
+// 	if (!cmd) throw new Error(_('Unknown command: %s', args));
+
+
+// 	let usage = cmd.usage.split(' ');
+// 	let commandArgs = [];
+// 	usage.splice(0, 1);
+// 	for (let i = 0; i < usage.length; i++) {
+// 		let u = usage[i].trim();
+// 		if (u == '') continue;
+
+// 		let required = false;
+
+// 		if (u.length >= 3 && u[0] == '<' && u[u.length - 1] == '>') {
+// 			required = true;
+// 			u = u.substr(1, u.length - 2);
+// 		}
+
+// 		if (u.length >= 3 && u[0] == '[' && u[u.length - 1] == ']') {
+// 			u = u.substr(1, u.length - 2);
+// 		}
+
+// 		if (required && !results['_'].length) throw new Error(_('Missing argument: %s', args));
+
+// 		if (!results['_'].length) break;
+
+// 		console.info(u);
+
+// 		commandArgs[u] = results['_'].splice(0, 1);
+// 	}
+
+// 	console.info(commandArgs);
+
+
+// 	// usage: 'import-enex <file> [notebook]',
+// 	// description: _('Imports en Evernote notebook file (.enex file).'),
+// 	// options: [
+// 	// 	['--fuzzy-matching', 'For debugging purposes. Do not use.'],
+// 	// ],
+
+// }
+
 async function synchronizer(syncTarget) {
 	if (synchronizers_[syncTarget]) return synchronizers_[syncTarget];
 
@@ -580,53 +617,49 @@ function cmdPromptConfirm(commandInstance, message) {
 	});
 }
 
-// Handles the initial arguments passed to main script and
+// Handles the initial flags passed to main script and
-// route them to the "root" command.
+// returns the remaining args.
-function handleStartArgs(argv) {
+async function handleStartFlags(argv) {
-	return new Promise((resolve, reject) => {
+	argv = argv.slice(0);
-		while (true) {
+	argv.splice(0, 2); // First arguments are the node executable, and the node JS file
+
+	while (argv.length) {
+		let arg = argv[0];
+		let nextArg = argv.length >= 2 ? argv[1] : null;
+		
+		if (arg == '--profile') {
+			if (!nextArg) {
+				throw new Error(_('Usage: --profile <dir-path>'));
+			}
+			initArgs.profileDir = nextArg;
 			argv.splice(0, 2);
-
+			continue;
-			if (argv[0] == '--profile') {
-				argv.splice(0, 1);
-				if (!argv.length) throw new Error(_('Profile path is missing'));
-				initArgs.profileDir = argv[0];
-				argv.splice(0, 1);
-			} else if (argv[0][0] === '-') {
-				throw new Error(_('Unknown flag: "%s"', argv[0]));
 		}
 
-			if (!argv.length || argv[0][0] != '-') {
+		if (arg.length && arg[0] == '-') {
-				resolve(argv);
+			throw new Error(_('Unknown flag: %s', arg));
+		} else {
+			break;
+		}
 	}
 
-			return;
+	return argv;
-
-			// if (argv && argv.length >= 3 && argv[2][0] == '-') {
-			// 	const startParams = vorpal.parse(argv, { use: 'minimist' });
-			// 	const cmd = commandByName('root');
-			// 	cmd.action(startParams, (newArgs) => {
-			// 		console.info(newArgs);
-			// 		resolve();
-			// 	});
-			// } else {
-			// 	console.info(argv);
-			// 	resolve();
-			// }
-
 }
-		// if (argv && argv.length >= 3 && argv[2][0] == '-') {
+
-		// 	const startParams = vorpal.parse(argv, { use: 'minimist' });
+function escapeShellArg(arg) {
-		// 	const cmd = commandByName('root');
+	if (arg.indexOf('"') >= 0 && arg.indexOf("'") >= 0) throw new Error(_('Command line argument "%s" contains both quotes and double-quotes - aborting.', arg)); // Hopeless case
-		// 	cmd.action(startParams, (newArgs) => {
+	let quote = '"';
-		// 		console.info(newArgs);
+	if (arg.indexOf('"') >= 0) quote = "'";
-		// 		resolve();
+	if (arg.indexOf(' ') >= 0 || arg.indexOf("\t") >= 0) return quote + arg + quote;
-		// 	});
+	return arg;
-		// } else {
+}
-		// 	console.info(argv);
+
-		// 	resolve();
+function shellArgsToString(args) {
-		// }
+	let output = [];
-	});
+	for (let i = 0; i < args.length; i++) {
+		output.push(escapeShellArg(args[i]));
+	}
+	return output.join(' ');
 }
 
 process.stdin.on('keypress', (_, key) => {
@@ -645,7 +678,6 @@ const vorpal = require('vorpal')();
 async function main() {
 	for (let commandIndex = 0; commandIndex < commands.length; commandIndex++) {
 		let c = commands[commandIndex];
-		if (c.usage == 'root') continue;
 		let o = vorpal.command(c.usage, c.description);
 		if (c.options) {
 			for (let i = 0; i < c.options.length; i++) {
@@ -669,7 +701,8 @@ async function main() {
 
 	vorpal.history('net.cozic.joplin'); // Enables persistent history
 
-	await handleStartArgs(process.argv);
+	let argv = process.argv;
+	argv = await handleStartFlags(argv);
 
 	const profileDir = initArgs.profileDir ? initArgs.profileDir : os.homedir() + '/.config/' + Setting.value('appName');
 	const resourceDir = profileDir + '/resources';
@@ -704,10 +737,19 @@ async function main() {
 	if (!activeFolder) activeFolder = await Folder.defaultFolder();
 	if (!activeFolder) activeFolder = await Folder.createDefaultFolder();
 	if (!activeFolder) throw new Error(_('No default notebook is defined and could not create a new one. The database might be corrupted, please delete it and try again.'));
+	Setting.setValue('activeFolderId', activeFolder.id);
 
-	if (activeFolder) await execCommand('cd', { 'notebook': activeFolder.title }); // Use execCommand() so that no history entry is created
+	await execCommand('cd', { 'notebook': activeFolder.title }); // Use execCommand() so that no history entry is created
-
 	vorpal.delimiter(promptString()).show();
+
+	// If we still have arguments, pass it to Vorpal and exit
+	if (argv.length) {
+		let cmd = shellArgsToString(argv);
+		vorpal.log(_('Executing: %s', cmd));
+		await vorpal.exec(cmd);
+		await vorpal.exec('exit');
+		return;
+	}
 }
 
 main().catch((error) => {

CliClient/package.json

@@ -1,6 +1,6 @@
 {
   "name": "joplin-cli",
-  "version": "0.8.11",
+  "version": "0.8.14",
   "bin": {
     "joplin": "./main.sh"
   },

CliClient/run.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
 set -e
 CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Downloads/desktop/afaire.enex afaire
+bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes
+#bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Desktop/afaire.enex afaire
+#bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Desktop/Laurent.enex laurent

CliClient/tests/synchronizer.js

@@ -20,6 +20,7 @@ async function allItems() {
 async function localItemsSameAsRemote(locals, expect) {
 	try {
 		let files = await fileApi().list();
+		files = files.items;
 		expect(locals.length).toBe(files.length);
 
 		for (let i = 0; i < locals.length; i++) {
@@ -116,7 +117,6 @@ describe('Synchronizer', function() {
 		await synchronizer().start();
 
 		let all = await allItems();
-		let files = await fileApi().list();
 
 		await localItemsSameAsRemote(all, expect);
 
@@ -227,6 +227,7 @@ describe('Synchronizer', function() {
 		await synchronizer().start();
 
 		let files = await fileApi().list();
+		files = files.items;
 
 		expect(files.length).toBe(1);
 		expect(files[0].path).toBe(Folder.systemPath(folder1));

lib/file-api-driver-local.js

@@ -53,7 +53,7 @@ class FileApiDriverLocal {
 		});
 	}
 
-	list(path) {
+	list(path, options) {
 		return new Promise((resolve, reject) => {
 			fs.readdir(path, (error, items) => {
 				if (error) {
@@ -75,7 +75,11 @@ class FileApiDriverLocal {
 
 				return promiseChain(chain).then((results) => {
 					if (!results) results = [];
-					resolve(results);
+					resolve({
+						items: results
+						hasMore: false,
+						context: null,
+					});
 				}).catch((error) => {
 					reject(error);
 				});

lib/file-api-driver-memory.js

@@ -41,7 +41,7 @@ class FileApiDriverMemory {
 		return Promise.resolve();
 	}
 
-	list(path) {
+	list(path, options) {
 		let output = [];
 
 		for (let i = 0; i < this.items_.length; i++) {
@@ -57,7 +57,11 @@ class FileApiDriverMemory {
 			}
 		}
 
-		return Promise.resolve(output);
+		return Promise.resolve({
+			items: output,
+			hasMore: false,
+			context: null,
+		});
 	}
 
 	get(path) {

lib/file-api-driver-onedrive.js

@@ -45,7 +45,7 @@ class FileApiDriverOneDrive {
 		try {
 			item = await this.api_.execJson('GET', this.makePath_(path), this.itemFilter_());
 		} catch (error) {
-			if (error.error.code == 'itemNotFound') return null;
+			if (error.code == 'itemNotFound') return null;
 			throw error;
 		}
 		return item;
@@ -67,9 +67,22 @@ class FileApiDriverOneDrive {
 		return this.makeItem_(item);
 	}
 
-	async list(path) {
+	async list(path, options = null) {
-		let items = await this.api_.execJson('GET', this.makePath_(path) + ':/children', this.itemFilter_());
+		let query = this.itemFilter_();
-		return this.makeItems_(items.value);
+		let url = this.makePath_(path) + ':/children';
+
+		if (options.context) {
+			query = null;
+			url = options.context;
+		}
+
+		let r = await this.api_.execJson('GET', url, query);
+
+		return {
+			hasMore: !!r['@odata.nextLink'],
+			items: this.makeItems_(r.value),
+			context: r["@odata.nextLink"],
+		}
 	}
 
 	async get(path) {
@@ -77,7 +90,7 @@ class FileApiDriverOneDrive {
 		try {
 			content = await this.api_.execText('GET', this.makePath_(path) + ':/content');
 		} catch (error) {
-			if (error.error.code == 'itemNotFound') return null;
+			if (error.code == 'itemNotFound') return null;
 			throw error;
 		}
 		return content;

lib/file-api.js

@@ -27,17 +27,19 @@ class FileApi {
 	list(path = '', options = null) {
 		if (!options) options = {};
 		if (!('includeHidden' in options)) options.includeHidden = false;
+		if (!('context' in options)) options.context = null;
 
-		this.logger().debug('list');
+		this.logger().debug('list ' + this.baseDir_);
-		return this.driver_.list(this.baseDir_).then((items) => {
+
+		return this.driver_.list(this.baseDir_, options).then((result) => {
 			if (!options.includeHidden) {
 				let temp = [];
-				for (let i = 0; i < items.length; i++) {
+				for (let i = 0; i < result.items.length; i++) {
-					if (!isHidden(items[i].path)) temp.push(items[i]);
+					if (!isHidden(result.items[i].path)) temp.push(result.items[i]);
 				}
-				items = temp;
+				result.items = temp;
 			}
-			return items;
+			return result;
 		});
 	}
 

lib/onedrive-api.js

@@ -67,6 +67,20 @@ class OneDriveApi {
 		return 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize?' + stringify(query);
 	}
 
+	oneDriveErrorResponseToError(errorResponse) {
+		if (!errorResponse) return new Error('Undefined error');
+
+		if (errorResponse.error) {
+			let e = errorResponse.error;
+			let output = new Error(e.message);
+			if (e.code) output.code = e.code;
+			if (e.innerError) output.innerError = e.innerError;
+			return output;
+		} else { 
+			return new Error(JSON.stringify(errorResponse));
+		}
+	}
+
 	async exec(method, path, query = null, data = null, options = null) {
 		method = method.toUpperCase();
 
@@ -82,26 +96,42 @@ class OneDriveApi {
 			if (data) data = JSON.stringify(data);
 		}
 
-		let url = 'https://graph.microsoft.com/v1.0' + path;
+		let url = path;
 
-		if (query) url += '?' + stringify(query);
+		// In general, `path` contains a path relative to the base URL, but in some
+		// cases the full URL is provided (for example, when it's a URL that was
+		// retrieved from the API).
+		if (url.indexOf('https://') !== 0) url = 'https://graph.microsoft.com/v1.0' + path;
+
+		if (query) {
+			url += url.indexOf('?') < 0 ? '?' : '&';
+			url += stringify(query);
+		}
 
 		if (data) options.body = data;
 
-		// console.info(method + ' ' + url);
+		// Rare error (one Google hit) - maybe repeat the request when it happens?
-		// console.info(data);
+
+		// { error:
+		//    { code: 'generalException',
+		//      message: 'An error occurred in the data store.',
+		//      innerError:
+		//       { 'request-id': 'b4310552-c18a-45b1-bde1-68e2c2345eef',
+		//         date: '2017-06-29T00:15:50' } } }
 
 		for (let i = 0; i < 5; i++) {
 			options.headers['Authorization'] = 'bearer ' + this.token();
 
 			let response = await fetch(url, options);
 			if (!response.ok) {
-				let error = await response.json();
+				let errorResponse = await response.json();
+				let error = this.oneDriveErrorResponseToError(errorResponse);
 
-				if (error && error.error && error.error.code == 'InvalidAuthenticationToken') {
+				if (error.code == 'InvalidAuthenticationToken') {
 					await this.refreshAccessToken();
 					continue;
 				} else {
+					error.request = method + ' ' + url + ' ' + JSON.stringify(query) + ' ' + JSON.stringify(data) + ' ' + JSON.stringify(options);
 					throw error;
 				}
 			}

lib/synchronizer.js

@@ -104,6 +104,7 @@ class Synchronizer {
 			noteConflict: 0,
 		};
 
+		try {
 			await this.createWorkDir();
 
 			let donePaths = [];
@@ -224,8 +225,11 @@ class Synchronizer {
 			// or handled as conflicts, so no conflict is possible after this.
 
 			let remoteIds = [];
-		let remotes = await this.api().list();
+			let context = null;
 
+			while (true) {
+				let listResult = await this.api().list('', { context: context });
+				let remotes = listResult.items;
 				for (let i = 0; i < remotes.length; i++) {
 					let remote = remotes[i];
 					let path = remote.path;
@@ -271,6 +275,10 @@ class Synchronizer {
 					report[action]++;
 				}
 
+				if (!listResult.hasMore) break;
+				context = listResult.context;
+			}
+
 			// ------------------------------------------------------------------------
 			// Search, among the local IDs, those that don't exist remotely, which
 			// means the item has been deleted.
@@ -285,13 +293,15 @@ class Synchronizer {
 					report['deleteLocal']++;
 				}
 			}
+		} catch (error) {
+			this.logger().error(error);
+			throw error;
+		}
 
 		this.logger().info('Synchronization complete [' + synchronizationId + ']:');
 		await this.logSyncSummary(report);
 
 		this.state_ = 'idle';
-
-		return Promise.resolve();
 	}
 
 }