From f96848a6bfab659ea330ba05a5a5d61df4082791 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Fri, 23 Jun 2017 18:51:02 +0000 Subject: [PATCH] Make create and update sync operations atomic --- CliClient/app/main.js | 9 +++--- CliClient/run_test.sh | 2 +- ReactNativeClient/src/file-api.js | 41 ++++++++++++++------------- ReactNativeClient/src/onedrive-api.js | 26 +++++++++++++---- ReactNativeClient/src/path-utils.js | 13 +++++++++ ReactNativeClient/src/synchronizer.js | 18 ++++++++++-- 6 files changed, 76 insertions(+), 33 deletions(-) create mode 100644 ReactNativeClient/src/path-utils.js diff --git a/CliClient/app/main.js b/CliClient/app/main.js index a264d4b6f..907487e95 100644 --- a/CliClient/app/main.js +++ b/CliClient/app/main.js @@ -53,11 +53,15 @@ async function main() { } driver.api().setAuth(auth); + driver.api().on('authRefreshed', (a) => { + Setting.setValue('sync.onedrive.auth', JSON.stringify(a)); + }); let appDir = await driver.api().appDirectory(); + console.info('App dir: ' + appDir); fileApi = new FileApi(appDir, driver); } else { - throw new Error('Unknown backend: ' . remoteBackend); + throw new Error('Unknown backend: ' + remoteBackend); } synchronizer_ = new Synchronizer(db, fileApi); @@ -65,9 +69,6 @@ async function main() { return synchronizer_; } - let s = await synchronizer(); - return; - function switchCurrentFolder(folder) { currentFolder = folder; updatePrompt(); diff --git a/CliClient/run_test.sh b/CliClient/run_test.sh index 9c2571727..787574724 100755 --- a/CliClient/run_test.sh +++ b/CliClient/run_test.sh @@ -5,5 +5,5 @@ rm -f "$CLIENT_DIR/tests-build/src" mkdir -p "$CLIENT_DIR/tests-build/data" ln -s "$CLIENT_DIR/build/src" "$CLIENT_DIR/tests-build" -npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js +#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js \ No newline at end of file diff --git a/ReactNativeClient/src/file-api.js b/ReactNativeClient/src/file-api.js index f941ce8a8..d00fb6905 100644 --- a/ReactNativeClient/src/file-api.js +++ b/ReactNativeClient/src/file-api.js @@ -1,4 +1,4 @@ -import { promiseChain } from 'src/promise-utils.js'; +import { isHidden } from 'src/path-utils.js'; class FileApi { @@ -35,20 +35,21 @@ class FileApi { return output; } - // listDirectories() { - // return this.driver_.list(this.fullPath_('')).then((items) => { - // let output = []; - // for (let i = 0; i < items.length; i++) { - // if (items[i].isDir) output.push(this.scopeItemToBaseDir_(items[i])); - // } - // return output; - // }); - // } + list(path = '', options = null) { + if (!options) options = {}; + if (!('includeHidden' in options)) options.includeHidden = false; - list() { this.dlog('list'); return this.driver_.list(this.baseDir_).then((items) => { - return this.scopeItemsToBaseDir_(items); + items = this.scopeItemsToBaseDir_(items); + if (!options.includeHidden) { + let temp = []; + for (let i = 0; i < items.length; i++) { + if (!isHidden(items[i].path)) temp.push(items[i]); + } + items = temp; + } + return items; }); } @@ -57,10 +58,10 @@ class FileApi { return this.driver_.setTimestamp(this.fullPath_(path), timestamp); } - // mkdir(path) { - // this.dlog('delete ' + path); - // return this.driver_.mkdir(this.fullPath_(path)); - // } + mkdir(path) { + this.dlog('delete ' + path); + return this.driver_.mkdir(this.fullPath_(path)); + } stat(path) { this.dlog('stat ' + path); @@ -86,10 +87,10 @@ class FileApi { return this.driver_.delete(this.fullPath_(path)); } - // move(oldPath, newPath) { - // this.dlog('move ' + path); - // return this.driver_.move(this.fullPath_(oldPath), this.fullPath_(newPath)); - // } + move(oldPath, newPath) { + this.dlog('move ' + oldPath + ' => ' + newPath); + return this.driver_.move(this.fullPath_(oldPath), this.fullPath_(newPath)); + } format() { return this.driver_.format(); diff --git a/ReactNativeClient/src/onedrive-api.js b/ReactNativeClient/src/onedrive-api.js index ece92ebb1..4c84dca17 100644 --- a/ReactNativeClient/src/onedrive-api.js +++ b/ReactNativeClient/src/onedrive-api.js @@ -12,6 +12,20 @@ class OneDriveApi { this.clientId_ = clientId; this.clientSecret_ = clientSecret; this.auth_ = null; + this.listeners_ = { + 'authRefreshed': [], + }; + } + + dispatch(eventName, param) { + let ls = this.listeners_[eventName]; + for (let i = 0; i < ls.length; i++) { + ls[i](param); + } + } + + on(eventName, callback) { + this.listeners_[eventName].push(callback); } tokenBaseUrl() { @@ -77,13 +91,15 @@ class OneDriveApi { console.info(method + ' ' + url); console.info(data); - while (true) { + for (let i = 0; i < 5; i++) { options.headers['Authorization'] = 'bearer ' + this.token(); let response = await fetch(url, options); if (!response.ok) { let error = await response.json(); + console.info(error); + if (error && error.error && error.error.code == 'InvalidAuthenticationToken') { await this.refreshAccessToken(); continue; @@ -94,6 +110,8 @@ class OneDriveApi { return response; } + + throw new Error('Could not execute request after multiple attempts: ' + method + ' ' + url); } async execJson(method, path, query, data) { @@ -133,11 +151,7 @@ class OneDriveApi { this.auth_ = await response.json(); - // POST https://login.microsoftonline.com/common/oauth2/v2.0/token - // Content-Type: application/x-www-form-urlencoded - - // client_id={client_id}&redirect_uri={redirect_uri}&client_secret={client_secret} - // &refresh_token={refresh_token}&grant_type=refresh_token + this.dispatch('authRefreshed', this.auth_); } async oauthDance() { diff --git a/ReactNativeClient/src/path-utils.js b/ReactNativeClient/src/path-utils.js new file mode 100644 index 000000000..d21f9d418 --- /dev/null +++ b/ReactNativeClient/src/path-utils.js @@ -0,0 +1,13 @@ +function basename(path) { + if (!path) throw new Error('Path is empty'); + let s = path.split('/'); + return s[s.length - 1]; +} + +function isHidden(path) { + let b = basename(path); + if (!b.length) throw new Error('Path empty or not a valid path: ' + path); + return b[0] === '.'; +} + +export { basename, isHidden }; \ No newline at end of file diff --git a/ReactNativeClient/src/synchronizer.js b/ReactNativeClient/src/synchronizer.js index 210b68bac..660a9c952 100644 --- a/ReactNativeClient/src/synchronizer.js +++ b/ReactNativeClient/src/synchronizer.js @@ -13,6 +13,7 @@ class Synchronizer { constructor(db, api) { this.db_ = db; this.api_ = api; + this.syncDirName_ = '.sync'; } db() { @@ -23,12 +24,20 @@ class Synchronizer { return this.api_; } + async createWorkDir() { + if (this.syncWorkDir_) return this.syncWorkDir_; + let dir = await this.api().mkdir(this.syncDirName_); + return this.syncDirName_; + } + async start() { // ------------------------------------------------------------------------ // First, find all the items that have been changed since the // last sync and apply the changes to remote. // ------------------------------------------------------------------------ + await this.createWorkDir(); + let donePaths = []; while (true) { let result = await BaseItem.itemsThatNeedSync(); @@ -69,8 +78,12 @@ class Synchronizer { if (action == 'createRemote' || action == 'updateRemote') { - await this.api().put(path, content); - await this.api().setTimestamp(path, local.updated_time); + // Make the operation atomic by doing the work on a copy of the file + // and then copying it back to the original location. + let tempPath = this.syncDirName_ + '/' + path; + await this.api().put(tempPath, content); + await this.api().setTimestamp(tempPath, local.updated_time); + await this.api().move(tempPath, path); await ItemClass.save({ id: local.id, sync_time: time.unixMs(), type_: local.type_ }, { autoTimestamp: false }); @@ -137,6 +150,7 @@ class Synchronizer { for (let i = 0; i < remotes.length; i++) { let remote = remotes[i]; let path = remote.path; + remoteIds.push(BaseItem.pathToId(path)); if (donePaths.indexOf(path) > 0) continue;