Allow cancelling sync and fixed resource issue

CliClient/app/main.js

@@ -28,6 +28,7 @@ import { vorpalUtils } from 'vorpal-utils.js';
 import { reg } from 'lib/registry.js';
 import { FsDriverNode } from './fs-driver-node.js';
 import { filename, basename } from 'lib/path-utils.js';
+import { shim } from 'lib/shim.js';
 import { _ } from 'lib/locale.js';
 import os from 'os';
 import fs from 'fs-extra';
@@ -521,7 +522,7 @@ commands.push({
 				if (report.remotesToDelete) line.push(_('Remote items to delete: %d/%d.', report.deleteRemote, report.remotesToDelete));
 				if (report.localsToUdpate) line.push(_('Items to download: %d/%d.', report.createLocal + report.updateLocal, report.localsToUdpate));
 				if (report.localsToDelete) line.push(_('Local items to delete: %d/%d.', report.deleteLocal, report.localsToDelete));
-				vorpalUtils.redraw(line.join(' '));
+				if (line.length) vorpalUtils.redraw(line.join(' '));
 			},
 			onMessage: (msg) => {
 				vorpalUtils.redrawDone();
@@ -549,6 +550,12 @@ commands.push({
 		this.log(_('Done.'));
 		end();
 	},
+	cancel: async function() {
+		vorpalUtils.redrawDone();
+		this.log(_('Cancelling...'));
+		let sync = await synchronizer(Setting.value('sync.target'));
+		sync.cancel();
+	},
 });
 
 commands.push({
@@ -896,9 +903,74 @@ const vorpal = require('vorpal')();
 vorpalUtils.initialize(vorpal);
 
 async function main() {
+
+	shim.fetchBlob = async function(url, options) {
+		if (!options || !options.path) throw new Error('fetchBlob: target file path is missing');
+		if (!options.method) options.method = 'GET';
+
+		const urlParse = require('url').parse;
+
+		url = urlParse(url.trim());
+		const http = url.protocol.toLowerCase() == 'http:' ? require('follow-redirects').http : require('follow-redirects').https;
+		const headers = options.headers ? options.headers : {};
+		const method = options.method ? options.method : 'GET';
+		if (method != 'GET') throw new Error('Only GET is supported');
+		const filePath = options.path;
+
+		function makeResponse(response) {
+			const output = {
+				ok: response.statusCode < 400,
+				path: filePath,
+				text: () => { return response.statusMessage; },
+				json: () => { return ''; },
+				status: response.statusCode,
+				headers: response.headers,
+			}
+			console.info(output);
+			return output;
+		}
+
+		const requestOptions = {
+			protocol: url.protocol,
+			host: url.host,
+			port: url.port,
+			method: method,
+			path: url.path + (url.query ? '?' + url.query : ''),
+			headers: headers,
+		};
+
+		return new Promise((resolve, reject) => {
+			try {
+				// Note: relative paths aren't supported
+				const file = fs.createWriteStream(filePath);
+
+				const request = http.get(requestOptions, function(response) {
+					response.pipe(file);
+
+					file.on('finish', function() {
+						console.info('FINISH');
+						file.close(() => {
+							console.info('FINISH CLOSE');
+							resolve(makeResponse(response));
+						});
+					});
+				})
+
+				request.on('error', function(error) {
+					fs.unlink(filePath);
+					reject(error);
+				});
+			} catch(error) {
+				fs.unlink(filePath);
+				reject(error);
+			}
+		});
+	}
+
 	for (let commandIndex = 0; commandIndex < commands.length; commandIndex++) {
 		let c = commands[commandIndex];
 		let o = vorpal.command(c.usage, c.description);
+
 		if (c.options) {
 			for (let i = 0; i < c.options.length; i++) {
 				let options = c.options[i];
@@ -906,16 +978,23 @@ async function main() {
 				if (options.length == 3) o.option(options[0], options[1], options[2]);
 			}
 		}
+
 		if (c.aliases) {
 			for (let i = 0; i < c.aliases.length; i++) {
 				o.alias(c.aliases[i]);
 			}
 		}
+
 		if (c.autocomplete) {
 			o.autocomplete({
 				data: c.autocomplete,
 			});
 		}
+
+		if (c.cancel) {
+			o.cancel(c.cancel);
+		}
+
 		o.action(c.action);
 	}
 
@@ -936,14 +1015,15 @@ async function main() {
 	logger.addTarget('file', { path: profileDir + '/log.txt' });
 	logger.setLevel(logLevel);
 
+	reg.setLogger(logger);
+
 	dbLogger.addTarget('file', { path: profileDir + '/log-database.txt' });
 	dbLogger.setLevel(logLevel);
 
 	syncLogger.addTarget('file', { path: profileDir + '/log-sync.txt' });
 	syncLogger.setLevel(logLevel);
 
-	logger.info(sprintf('Starting %s %s...', packageJson.name, packageJson.version));
-	logger.info('Environment: ' + Setting.value('env'));
+	logger.info(sprintf('Starting %s %s (%s)...', packageJson.name, packageJson.version, Setting.value('env')));
 	logger.info('Profile directory: ' + profileDir);
 
 	// That's not good, but it's to avoid circular dependency issues

CliClient/app/main_launcher.js

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+
+// Because all the files in the "lib" directory are included as "lib/file.js" it
+// means "lib" must be in NODE_PATH, however modifying the global NODE_PATH
+// variable would be messy. So instead, the path is set temporarily just before running
+// the app. To do this, this wrapper is needed.
+// See https://gist.github.com/branneman/8048520
+// Original wrapper code from https://gist.github.com/branneman/8775568
+
+'use strict';
+
+var spawn = require('child_process').spawn;
+
+var args = ['main.js'];
+var processArgs = process.argv.splice(2);
+args = args.concat(processArgs);
+
+var opt = {
+	cwd: __dirname,
+	env: (function() {
+		process.env.NODE_PATH = '.'; // Enables require() calls relative to the cwd :)
+		return process.env;
+	}()),
+	stdio: [process.stdin, process.stdout, process.stderr]
+};
+
+var app = spawn(process.execPath, args, opt);
+
+// Pass on the exit code
+app.on('close', (code) => {
+	process.exit(code);
+});

CliClient/app/vorpal-utils.js

@@ -13,6 +13,15 @@ function initialize(vorpal) {
 	vorpal_ = vorpal;
 }
 
+function redrawEnabled() {
+	// Always disabled for now - doesn't play well with command.cancel()
+	// function (it makes the whole app quit instead of just the 
+	// current command).
+	return false;
+
+	return redrawEnabled_;
+}
+
 function setRedrawEnabled(v) {
 	redrawEnabled_ = v;
 }
@@ -22,7 +31,7 @@ function setStackTraceEnabled(v) {
 }
 
 function redraw(s) {
-	if (!redrawEnabled_) {
+	if (!redrawEnabled()) {
 		const now = time.unixMs();
 		if (now - redrawLastUpdateTime_ > 1000) {
 			if (vorpal_.activeCommand) {
@@ -45,7 +54,7 @@ function redraw(s) {
 function redrawDone() {
 	if (!redrawStarted_) return;
 
-	if (!redrawEnabled_) {
+	if (!redrawEnabled()) {
 		if (redrawLastLog_) {
 			if (vorpal_.activeCommand) {
 				vorpal_.activeCommand.log(redrawLastLog_);

CliClient/install.sh

@@ -5,8 +5,8 @@ CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 npm version patch
 bash $CLIENT_DIR/build.sh
 cp "$CLIENT_DIR/package.json" build/
-cp "$CLIENT_DIR/../lib/package.json" build/lib
-cp "$CLIENT_DIR/app/main.sh" build/
+cp "$CLIENT_DIR/../ReactNativeClient/lib/package.json" build/lib
+#cp "$CLIENT_DIR/app/main.sh" build/
 cd "$CLIENT_DIR/build"
 sudo npm install -g --save
 cd -

CliClient/package.json

@@ -7,13 +7,14 @@
     "url": "https://github.com/laurent22/joplin"
   },
   "url": "git://github.com/laurent22/joplin.git",
-  "version": "0.8.27",
+  "version": "0.8.29",
   "bin": {
-    "joplin": "./main.sh"
+    "joplin": "./main_launcher.js"
   },
   "dependencies": {
     "app-module-path": "^2.2.0",
     "babel-plugin-transform-runtime": "^6.23.0",
+    "follow-redirects": "^1.2.4",
     "form-data": "^2.1.4",
     "fs-extra": "^3.0.1",
     "jssha": "^2.3.0",

CliClient/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
 CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes2 --stack-trace-enabled --log-level debug --redraw-disabled --env dev "$@"
+bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes2 --stack-trace-enabled --log-level debug --env dev "$@"
 #bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Desktop/afaire.enex afaire
 #bash $CLIENT_DIR/build.sh && NODE_PATH="$CLIENT_DIR/build/" node build/main.js --profile ~/Temp/TestNotes import-enex --fuzzy-matching /home/laurent/Desktop/Laurent.enex laurent

ReactNativeClient/android/app/build.gradle

@@ -90,8 +90,8 @@ android {
 		applicationId "net.cozic.joplin"
 		minSdkVersion 16
 		targetSdkVersion 22
-		versionCode 2
-		versionName "0.8.0"
+		versionCode 3
+		versionName "0.8.1"
 		ndk {
 			abiFilters "armeabi-v7a", "x86"
 		}

ReactNativeClient/lib/onedrive-api.js

@@ -1,6 +1,7 @@
 import { shim } from 'lib/shim.js';
 import { stringify } from 'query-string';
 import { time } from 'lib/time-utils.js';
+import { Logger } from 'lib/logger.js'
 
 class OneDriveApi {
 
@@ -16,6 +17,15 @@ class OneDriveApi {
 		this.listeners_ = {
 			'authRefreshed': [],
 		};
+		this.logger_ = new Logger();
+	}
+
+	setLogger(l) {
+		this.logger_ = l;
+	}
+
+	logger() {
+		return this.logger_;
 	}
 
 	isPublic() {
@@ -161,17 +171,19 @@ class OneDriveApi {
 				let error = this.oneDriveErrorResponseToError(errorResponse);
 
 				if (error.code == 'InvalidAuthenticationToken' || error.code == 'unauthenticated') {
+					this.logger().info('Token expired: refreshing...');
 					await this.refreshAccessToken();
 					continue;
 				} else if (error && ((error.error && error.error.code == 'generalException') || (error.code == 'generalException'))) {
 					// Rare error (one Google hit) - I guess the request can be repeated
-					
 					// { error:
 					//    { code: 'generalException',
 					//      message: 'An error occurred in the data store.',
 					//      innerError:
 					//       { 'request-id': 'b4310552-c18a-45b1-bde1-68e2c2345eef',
 					//         date: '2017-06-29T00:15:50' } } }
+					this.logger().info('Got error below - retrying...');
+					this.logger().info(error);
 					await time.msleep(1000 * i);
 					continue;
 				} else if (error.code == 'EAGAIN') {
@@ -181,6 +193,8 @@ class OneDriveApi {
 					//   type: 'system',
 					//   errno: 'EAGAIN',
 					//   code: 'EAGAIN' }
+					this.logger().info('Got error below - retrying...');
+					this.logger().info(error);
 					await time.msleep(1000 * i);
 					continue;
 				} else {

ReactNativeClient/lib/registry.js

@@ -16,12 +16,17 @@ reg.logger = () => {
 	return reg.logger_;
 }
 
+reg.setLogger = (l) => {
+	reg.logger_ = l;
+}
+
 reg.oneDriveApi = () => {
 	if (reg.oneDriveApi_) return reg.oneDriveApi_;
 
 	const isPublic = Setting.value('appType') != 'cli';
 
 	reg.oneDriveApi_ = new OneDriveApi(parameters().oneDrive.id, parameters().oneDrive.secret, isPublic);
+	reg.oneDriveApi_.setLogger(reg.logger());
 
 	reg.oneDriveApi_.on('authRefreshed', (a) => {
 		reg.logger().info('Saving updated OneDrive auth.');

ReactNativeClient/lib/synchronizer.js

@@ -18,6 +18,7 @@ class Synchronizer {
 		this.resourceDirName_ = '.resource';
 		this.logger_ = new Logger();
 		this.appType_ = appType;
+		this.cancelling_ = false;
 	}
 
 	state() {
@@ -86,6 +87,17 @@ class Synchronizer {
 		return false;
 	}
 
+	cancel() {
+		if (this.cancelling_) return;
+		
+		this.logger().info('Cancelling synchronization...');
+		this.cancelling_ = true;
+	}
+
+	cancelling() {
+		return this.cancelling_;
+	}
+
 	async start(options = null) {
 		if (!options) options = {};
 		if (!options.onProgress) options.onProgress = function(o) {};
@@ -96,6 +108,7 @@ class Synchronizer {
 		}	
 
 		this.randomFailureChoice_ = Math.floor(Math.random() * 5);
+		this.cancelling_ = false;
 
 		// ------------------------------------------------------------------------
 		// First, find all the items that have been changed since the
@@ -131,6 +144,8 @@ class Synchronizer {
 
 			let donePaths = [];
 			while (true) {
+				if (this.cancelling()) break;
+
 				let result = await BaseItem.itemsThatNeedSync();
 				let locals = result.items;
 
@@ -138,6 +153,8 @@ class Synchronizer {
 				options.onProgress(report);
 
 				for (let i = 0; i < locals.length; i++) {
+					if (this.cancelling()) break;
+
 					let local = locals[i];
 					let ItemClass = BaseItem.itemClass(local);
 					let path = BaseItem.systemPath(local);
@@ -249,6 +266,8 @@ class Synchronizer {
 			report.remotesToDelete = deletedItems.length;
 			options.onProgress(report);
 			for (let i = 0; i < deletedItems.length; i++) {
+				if (this.cancelling()) break;
+
 				let item = deletedItems[i];
 				let path = BaseItem.systemPath(item.item_id)
 				this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted');
@@ -272,9 +291,13 @@ class Synchronizer {
 			let context = null;
 
 			while (true) {
+				if (this.cancelling()) break;
+
 				let listResult = await this.api().list('', { context: context });
 				let remotes = listResult.items;
 				for (let i = 0; i < remotes.length; i++) {
+					if (this.cancelling()) break;
+
 					let remote = remotes[i];
 					let path = remote.path;
 
@@ -319,14 +342,17 @@ class Synchronizer {
 						if (newContent.type_ == BaseModel.TYPE_RESOURCE && action == 'createLocal') {
 							let localResourceContentPath = Resource.fullPath(newContent);
 							let remoteResourceContentPath = this.resourceDirName_ + '/' + newContent.id;
-							if (this.appType_ == 'cli') {								
-								let remoteResourceContent = await this.api().get(remoteResourceContentPath, { encoding: 'binary' });
-								await Resource.setContent(newContent, remoteResourceContent);
-							} else if (this.appType_ == 'mobile') {
-								await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: 'file' });
-							} else {
-								throw new Error('Unknown appType: ' + this.appType_);
-							}
+
+							await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: 'file' });
+							
+							// if (this.appType_ == 'cli') {								
+							// 	let remoteResourceContent = await this.api().get(remoteResourceContentPath, { encoding: 'binary' });
+							// 	await Resource.setContent(newContent, remoteResourceContent);
+							// } else if (this.appType_ == 'mobile') {
+							// 	await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: 'file' });
+							// } else {
+							// 	throw new Error('Unknown appType: ' + this.appType_);
+							// }
 						}
 
 						await ItemClass.save(newContent, options);
@@ -352,18 +378,22 @@ class Synchronizer {
 
 			if (this.randomFailure(options, 4)) return;
 
-			let items = await BaseItem.syncedItems();
-			for (let i = 0; i < items.length; i++) {
-				let item = items[i];
-				if (remoteIds.indexOf(item.id) < 0) {
-					report.localsToDelete++;
-					options.onProgress(report);
-					this.logSyncOperation('deleteLocal', { id: item.id }, null, 'remote has been deleted');
+			if (!this.cancelling()) {
+				let items = await BaseItem.syncedItems();
+				for (let i = 0; i < items.length; i++) {
+					if (this.cancelling()) break;
 
-					let ItemClass = BaseItem.itemClass(item);
-					await ItemClass.delete(item.id, { trackDeleted: false });
-					report['deleteLocal']++;
-					options.onProgress(report);
+					let item = items[i];
+					if (remoteIds.indexOf(item.id) < 0) {
+						report.localsToDelete++;
+						options.onProgress(report);
+						this.logSyncOperation('deleteLocal', { id: item.id }, null, 'remote has been deleted');
+
+						let ItemClass = BaseItem.itemClass(item);
+						await ItemClass.delete(item.id, { trackDeleted: false });
+						report['deleteLocal']++;
+						options.onProgress(report);
+					}
 				}
 			}
 		} catch (error) {
@@ -371,6 +401,11 @@ class Synchronizer {
 			throw error;
 		}
 
+		if (this.cancelling()) {
+			this.logger().info('Synchronization was cancelled.');
+			this.cancelling_ = false;
+		}
+
 		this.logger().info('Synchronization complete [' + synchronizationId + ']:');
 		await this.logSyncSummary(report);
 

ReactNativeClient/start_server.bat

@@ -1,2 +1,3 @@
 @echo off
-npm start -- --reset-cache
+npm start -- --reset-cache
+rem "c:\Program Files (x86)\Yarn\bin\yarn.cmd" start -- --reset-cache