Make create and update sync operations atomic

CliClient/app/main.js

@@ -53,11 +53,15 @@ async function main() {
 			}
 
 			driver.api().setAuth(auth);
+			driver.api().on('authRefreshed', (a) => {
+				Setting.setValue('sync.onedrive.auth', JSON.stringify(a));
+			});
 
 			let appDir = await driver.api().appDirectory();
+			console.info('App dir: ' + appDir);
 			fileApi = new FileApi(appDir, driver);
 		} else {
-			throw new Error('Unknown backend: ' . remoteBackend);
+			throw new Error('Unknown backend: ' + remoteBackend);
 		}
 
 		synchronizer_ = new Synchronizer(db, fileApi);
@@ -65,9 +69,6 @@ async function main() {
 		return synchronizer_;
 	}
 
-	let s = await synchronizer();
-	return;
-
 	function switchCurrentFolder(folder) {
 		currentFolder = folder;
 		updatePrompt();

CliClient/run_test.sh

@@ -5,5 +5,5 @@ rm -f "$CLIENT_DIR/tests-build/src"
 mkdir -p "$CLIENT_DIR/tests-build/data"
 ln -s "$CLIENT_DIR/build/src" "$CLIENT_DIR/tests-build"
 
-npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js
+#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js
 npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js

ReactNativeClient/src/file-api.js

@@ -1,4 +1,4 @@
-import { promiseChain } from 'src/promise-utils.js';
+import { isHidden } from 'src/path-utils.js';
 
 class FileApi {
 
@@ -35,20 +35,21 @@ class FileApi {
 		return output;
 	}
 
-	// listDirectories() {
+	list(path = '', options = null) {
-	// 	return this.driver_.list(this.fullPath_('')).then((items) => {
+		if (!options) options = {};
-	// 		let output = [];
+		if (!('includeHidden' in options)) options.includeHidden = false;
-	// 		for (let i = 0; i < items.length; i++) {
-	// 			if (items[i].isDir) output.push(this.scopeItemToBaseDir_(items[i]));
-	// 		}
-	// 		return output;
-	// 	});
-	// }
 
-	list() {
 		this.dlog('list');
 		return this.driver_.list(this.baseDir_).then((items) => {
-			return this.scopeItemsToBaseDir_(items);
+			items = this.scopeItemsToBaseDir_(items);
+			if (!options.includeHidden) {
+				let temp = [];
+				for (let i = 0; i < items.length; i++) {
+					if (!isHidden(items[i].path)) temp.push(items[i]);
+				}
+				items = temp;
+			}
+			return items;
 		});
 	}
 
@@ -57,10 +58,10 @@ class FileApi {
 		return this.driver_.setTimestamp(this.fullPath_(path), timestamp);
 	}
 
-	// mkdir(path) {
+	mkdir(path) {
-	// 	this.dlog('delete ' + path);
+		this.dlog('delete ' + path);
-	// 	return this.driver_.mkdir(this.fullPath_(path));
+		return this.driver_.mkdir(this.fullPath_(path));
-	// }
+	}
 
 	stat(path) {
 		this.dlog('stat ' + path);
@@ -86,10 +87,10 @@ class FileApi {
 		return this.driver_.delete(this.fullPath_(path));
 	}
 
-	// move(oldPath, newPath) {
+	move(oldPath, newPath) {
-	// 	this.dlog('move ' + path);
+		this.dlog('move ' + oldPath + ' => ' + newPath);
-	// 	return this.driver_.move(this.fullPath_(oldPath), this.fullPath_(newPath));
+		return this.driver_.move(this.fullPath_(oldPath), this.fullPath_(newPath));
-	// }
+	}
 
 	format() {
 		return this.driver_.format();

ReactNativeClient/src/onedrive-api.js

@@ -12,6 +12,20 @@ class OneDriveApi {
 		this.clientId_ = clientId;
 		this.clientSecret_ = clientSecret;
 		this.auth_ = null;
+		this.listeners_ = {
+			'authRefreshed': [],
+		};
+	}
+
+	dispatch(eventName, param) {
+		let ls = this.listeners_[eventName];
+		for (let i = 0; i < ls.length; i++) {
+			ls[i](param);
+		}
+	}
+
+	on(eventName, callback) {
+		this.listeners_[eventName].push(callback);
 	}
 
 	tokenBaseUrl() {
@@ -77,13 +91,15 @@ class OneDriveApi {
 		console.info(method + ' ' + url);
 		console.info(data);
 
-		while (true) {
+		for (let i = 0; i < 5; i++) {
 			options.headers['Authorization'] = 'bearer ' + this.token();
 
 			let response = await fetch(url, options);
 			if (!response.ok) {
 				let error = await response.json();
 
+				console.info(error);
+
 				if (error && error.error && error.error.code == 'InvalidAuthenticationToken') {
 					await this.refreshAccessToken();
 					continue;
@@ -94,6 +110,8 @@ class OneDriveApi {
 
 			return response;
 		}
+
+		throw new Error('Could not execute request after multiple attempts: ' + method + ' ' + url);
 	}
 
 	async execJson(method, path, query, data) {
@@ -133,11 +151,7 @@ class OneDriveApi {
 
 		this.auth_ = await response.json();
 
-		// POST https://login.microsoftonline.com/common/oauth2/v2.0/token
+		this.dispatch('authRefreshed', this.auth_);
-		// Content-Type: application/x-www-form-urlencoded
-
-		// client_id={client_id}&redirect_uri={redirect_uri}&client_secret={client_secret}
-		// &refresh_token={refresh_token}&grant_type=refresh_token
 	}
 
 	async oauthDance() {

ReactNativeClient/src/path-utils.js

@@ -0,0 +1,13 @@
+function basename(path) {
+	if (!path) throw new Error('Path is empty');
+	let s = path.split('/');
+	return s[s.length - 1];
+}
+
+function isHidden(path) {
+	let b = basename(path);
+	if (!b.length) throw new Error('Path empty or not a valid path: ' + path);
+	return b[0] === '.';
+}
+
+export { basename, isHidden };

ReactNativeClient/src/synchronizer.js

@@ -13,6 +13,7 @@ class Synchronizer {
 	constructor(db, api) {
 		this.db_ = db;
 		this.api_ = api;
+		this.syncDirName_ = '.sync';
 	}
 
 	db() {
@@ -23,12 +24,20 @@ class Synchronizer {
 		return this.api_;
 	}
 
+	async createWorkDir() {
+		if (this.syncWorkDir_) return this.syncWorkDir_;
+		let dir = await this.api().mkdir(this.syncDirName_);
+		return this.syncDirName_;
+	}
+
 	async start() {
 		// ------------------------------------------------------------------------
 		// First, find all the items that have been changed since the
 		// last sync and apply the changes to remote.
 		// ------------------------------------------------------------------------
 
+		await this.createWorkDir();
+
 		let donePaths = [];
 		while (true) {
 			let result = await BaseItem.itemsThatNeedSync();
@@ -69,8 +78,12 @@ class Synchronizer {
 
 				if (action == 'createRemote' || action == 'updateRemote') {
 
-					await this.api().put(path, content);
+					// Make the operation atomic by doing the work on a copy of the file
-					await this.api().setTimestamp(path, local.updated_time);
+					// and then copying it back to the original location.
+					let tempPath = this.syncDirName_ + '/' + path;
+					await this.api().put(tempPath, content);
+					await this.api().setTimestamp(tempPath, local.updated_time);
+					await this.api().move(tempPath, path);
 
 					await ItemClass.save({ id: local.id, sync_time: time.unixMs(), type_: local.type_ }, { autoTimestamp: false });
 
@@ -137,6 +150,7 @@ class Synchronizer {
 		for (let i = 0; i < remotes.length; i++) {
 			let remote = remotes[i];
 			let path = remote.path;
+
 			remoteIds.push(BaseItem.pathToId(path));
 			if (donePaths.indexOf(path) > 0) continue;