From 927894e940470146f52320fd560b42857bb2f0a3 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Tue, 18 Jul 2017 19:57:49 +0000 Subject: [PATCH] OneDrive delta api --- CliClient/app/command-sync.js | 6 +- CliClient/locales/en_GB.po | 6 +- CliClient/locales/fr_FR.po | 6 +- CliClient/locales/joplin.pot | 6 +- .../lib/file-api-driver-onedrive.js | 53 ++++- ReactNativeClient/lib/file-api.js | 5 + ReactNativeClient/lib/models/setting.js | 1 + ReactNativeClient/lib/synchronizer.js | 189 +++++++++--------- 8 files changed, 169 insertions(+), 103 deletions(-) diff --git a/CliClient/app/command-sync.js b/CliClient/app/command-sync.js index 0cd00342d..a69d5625c 100644 --- a/CliClient/app/command-sync.js +++ b/CliClient/app/command-sync.js @@ -91,7 +91,11 @@ class Command extends BaseCommand { this.log(_('Starting synchronization...')); - await sync.start(options); + let context = Setting.value('sync.context'); + context = context ? JSON.parse(context) : {}; + options.context = context; + let newContext = await sync.start(options); + Setting.setValue('sync.context', JSON.stringify(newContext)); vorpalUtils.redrawDone(); await app().refreshCurrentFolder(); diff --git a/CliClient/locales/en_GB.po b/CliClient/locales/en_GB.po index d875f7c53..927d2026a 100644 --- a/CliClient/locales/en_GB.po +++ b/CliClient/locales/en_GB.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Joplin-CLI 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-07-18 16:33+0100\n" +"POT-Creation-Date: 2017-07-18 17:30+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -316,11 +316,11 @@ msgstr "" msgid "Starting synchronization..." msgstr "" -#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:99 +#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:103 msgid "Done." msgstr "" -#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:114 +#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:118 #: /media/veracrypt22/src/notes/ReactNativeClient/lib/synchronizer.js:60 msgid "Cancelling..." msgstr "" diff --git a/CliClient/locales/fr_FR.po b/CliClient/locales/fr_FR.po index 4de3b58e0..8ee9afad2 100644 --- a/CliClient/locales/fr_FR.po +++ b/CliClient/locales/fr_FR.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: Joplin-CLI 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-07-18 16:32+0100\n" +"POT-Creation-Date: 2017-07-18 17:28+0100\n" "PO-Revision-Date: 2017-07-18 13:27+0100\n" "Last-Translator: \n" "Language-Team: \n" @@ -342,11 +342,11 @@ msgstr "Impossible d'initialiser le synchroniseur." msgid "Starting synchronization..." msgstr "Commencement de la synchronisation..." -#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:99 +#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:103 msgid "Done." msgstr "Terminé." -#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:114 +#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:118 #: /media/veracrypt22/src/notes/ReactNativeClient/lib/synchronizer.js:60 msgid "Cancelling..." msgstr "Annulation..." diff --git a/CliClient/locales/joplin.pot b/CliClient/locales/joplin.pot index d875f7c53..927d2026a 100644 --- a/CliClient/locales/joplin.pot +++ b/CliClient/locales/joplin.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Joplin-CLI 1.0.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-07-18 16:33+0100\n" +"POT-Creation-Date: 2017-07-18 17:30+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -316,11 +316,11 @@ msgstr "" msgid "Starting synchronization..." msgstr "" -#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:99 +#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:103 msgid "Done." msgstr "" -#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:114 +#: /media/veracrypt22/src/notes/CliClient/app/command-sync.js:118 #: /media/veracrypt22/src/notes/ReactNativeClient/lib/synchronizer.js:60 msgid "Cancelling..." msgstr "" diff --git a/ReactNativeClient/lib/file-api-driver-onedrive.js b/ReactNativeClient/lib/file-api-driver-onedrive.js index 4afd936c0..857d7ac35 100644 --- a/ReactNativeClient/lib/file-api-driver-onedrive.js +++ b/ReactNativeClient/lib/file-api-driver-onedrive.js @@ -166,10 +166,55 @@ class FileApiDriverOneDrive { throw new Error('Not implemented'); } - // delta(path) { - // let response = await this.api_.exec('GET', this.makePath_(path) + ':/delta'); - // console.info(response); - // } + async delta(path, options = null) { + let output = { + context: {}, + items: [], + }; + + let context = options ? options.context : null; + + let url = null; + let query = null; + if (context) { + url = context; + } else { + url = this.makePath_(path) + ':/delta'; + query = this.itemFilter_(); + } + + while (true) { + let response = await this.api_.execJson('GET', url, query); + let items = this.makeItems_(response.value); + output.items = output.items.concat(items); + + if (response['@odata.nextLink']) { + url = response['@odata.nextLink']; + } else { + if (!response['@odata.deltaLink']) { + throw new Error('Delta link missing: ' + JSON.stringify(response)); + } + output.context = response['@odata.deltaLink']; + break; + } + } + + // https://dev.onedrive.com/items/view_delta.htm + // The same item may appear more than once in a delta feed, for various reasons. You should use the last occurrence you see. + // So remove any duplicate item from the array. + let temp = []; + let seenPaths = []; + for (let i = output.items.length - 1; i >= 0; i--) { + let item = output.items[i]; + if (seenPaths.indexOf(item.path) >= 0) continue; + temp.splice(0, 0, item); + seenPaths.push(item.path); + } + + output.items = temp; + + return output; + } } diff --git a/ReactNativeClient/lib/file-api.js b/ReactNativeClient/lib/file-api.js index b4cf5ea56..44cb17f99 100644 --- a/ReactNativeClient/lib/file-api.js +++ b/ReactNativeClient/lib/file-api.js @@ -92,6 +92,11 @@ class FileApi { return this.driver_.format(); } + delta(path, options = null) { + this.logger().debug('delta ' + this.fullPath_(path)); + return this.driver_.delta(this.fullPath_(path), options); + } + } export { FileApi }; \ No newline at end of file diff --git a/ReactNativeClient/lib/models/setting.js b/ReactNativeClient/lib/models/setting.js index 9432344b1..3704e0fc8 100644 --- a/ReactNativeClient/lib/models/setting.js +++ b/ReactNativeClient/lib/models/setting.js @@ -148,6 +148,7 @@ Setting.defaults_ = { 'sync.onedrive.auth': { value: '', type: 'string', public: false }, 'sync.filesystem.path': { value: '', type: 'string', public: true }, 'sync.target': { value: 'onedrive', type: 'string', public: true }, + 'sync.context': { value: '', type: 'string', public: false }, 'editor': { value: '', type: 'string', public: true }, 'locale': { value: 'en_GB', type: 'string', public: true }, 'aliases': { value: '', type: 'string', public: true }, diff --git a/ReactNativeClient/lib/synchronizer.js b/ReactNativeClient/lib/synchronizer.js index a284a9e0d..819aacab3 100644 --- a/ReactNativeClient/lib/synchronizer.js +++ b/ReactNativeClient/lib/synchronizer.js @@ -147,6 +147,8 @@ class Synchronizer { this.onProgress_ = options.onProgress ? options.onProgress : function(o) {}; this.progressReport_ = { errors: [] }; + let lastContext = options.context; + const syncTargetId = this.api().driver().syncTargetId(); if (this.state() != 'idle') { @@ -164,6 +166,8 @@ class Synchronizer { let synchronizationId = time.unixMs().toString(); + let outputContext = {}; + this.state_ = 'in_progress'; this.dispatch({ type: 'SYNC_STARTED' }); @@ -315,116 +319,121 @@ class Synchronizer { // At this point all the local items that have changed have been pushed to remote // or handled as conflicts, so no conflict is possible after this. - let remoteIds = []; - let context = null; + let deltaOptions = {}; + if (lastContext.delta) deltaOptions.context = lastContext.delta; + let listResult = await this.api().delta('', deltaOptions); + outputContext.delta = listResult.context; - while (true) { - if (this.cancelling()) break; + // let remoteIds = []; + // let context = null; - let listResult = await this.api().list('', { context: context }); - let remotes = listResult.items; - for (let i = 0; i < remotes.length; i++) { - if (this.cancelling()) break; + // while (true) { + // if (this.cancelling()) break; - let remote = remotes[i]; - let path = remote.path; + // let listResult = await this.api().list('', { context: context }); + // let remotes = listResult.items; + // for (let i = 0; i < remotes.length; i++) { + // if (this.cancelling()) break; - remoteIds.push(BaseItem.pathToId(path)); - if (donePaths.indexOf(path) > 0) continue; + // let remote = remotes[i]; + // let path = remote.path; - let action = null; - let reason = ''; - let local = await BaseItem.loadItemByPath(path); - if (!local) { - action = 'createLocal'; - reason = 'remote exists but local does not'; - } else { - if (remote.updated_time > local.updated_time) { - action = 'updateLocal'; - reason = sprintf('remote is more recent than local'); - } - } + // remoteIds.push(BaseItem.pathToId(path)); + // if (donePaths.indexOf(path) > 0) continue; - if (!action) continue; + // let action = null; + // let reason = ''; + // let local = await BaseItem.loadItemByPath(path); + // if (!local) { + // action = 'createLocal'; + // reason = 'remote exists but local does not'; + // } else { + // if (remote.updated_time > local.updated_time) { + // action = 'updateLocal'; + // reason = sprintf('remote is more recent than local'); + // } + // } - if (action == 'createLocal' || action == 'updateLocal') { - let content = await this.api().get(path); - if (content === null) { - this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path); - continue; - } - content = await BaseItem.unserialize(content); - let ItemClass = BaseItem.itemClass(content); + // if (!action) continue; - let newContent = Object.assign({}, content); - let options = { - autoTimestamp: false, - applyMetadataChanges: true, - nextQueries: BaseItem.updateSyncTimeQueries(syncTargetId, newContent, time.unixMs()), - }; - if (action == 'createLocal') options.isNew = true; + // if (action == 'createLocal' || action == 'updateLocal') { + // let content = await this.api().get(path); + // if (content === null) { + // this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path); + // continue; + // } + // content = await BaseItem.unserialize(content); + // let ItemClass = BaseItem.itemClass(content); - if (newContent.type_ == BaseModel.TYPE_RESOURCE && action == 'createLocal') { - let localResourceContentPath = Resource.fullPath(newContent); - let remoteResourceContentPath = this.resourceDirName_ + '/' + newContent.id; - await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: 'file' }); - } + // let newContent = Object.assign({}, content); + // let options = { + // autoTimestamp: false, + // applyMetadataChanges: true, + // nextQueries: BaseItem.updateSyncTimeQueries(syncTargetId, newContent, time.unixMs()), + // }; + // if (action == 'createLocal') options.isNew = true; - await ItemClass.save(newContent, options); + // if (newContent.type_ == BaseModel.TYPE_RESOURCE && action == 'createLocal') { + // let localResourceContentPath = Resource.fullPath(newContent); + // let remoteResourceContentPath = this.resourceDirName_ + '/' + newContent.id; + // await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: 'file' }); + // } - this.logSyncOperation(action, local, content, reason); - } else { - this.logSyncOperation(action, local, remote, reason); - } - } + // await ItemClass.save(newContent, options); - if (!listResult.hasMore) break; - context = listResult.context; - } + // this.logSyncOperation(action, local, content, reason); + // } else { + // this.logSyncOperation(action, local, remote, reason); + // } + // } - // ------------------------------------------------------------------------ - // Search, among the local IDs, those that don't exist remotely, which - // means the item has been deleted. - // ------------------------------------------------------------------------ + // if (!listResult.hasMore) break; + // context = listResult.context; + // } - if (this.randomFailure(options, 4)) return; + // // ------------------------------------------------------------------------ + // // Search, among the local IDs, those that don't exist remotely, which + // // means the item has been deleted. + // // ------------------------------------------------------------------------ - let localFoldersToDelete = []; + // if (this.randomFailure(options, 4)) return; - if (!this.cancelling()) { - let syncItems = await BaseItem.syncedItems(syncTargetId); - for (let i = 0; i < syncItems.length; i++) { - if (this.cancelling()) break; + // let localFoldersToDelete = []; - let syncItem = syncItems[i]; - if (remoteIds.indexOf(syncItem.item_id) < 0) { - if (syncItem.item_type == Folder.modelType()) { - localFoldersToDelete.push(syncItem); - continue; - } + // if (!this.cancelling()) { + // let syncItems = await BaseItem.syncedItems(syncTargetId); + // for (let i = 0; i < syncItems.length; i++) { + // if (this.cancelling()) break; - this.logSyncOperation('deleteLocal', { id: syncItem.item_id }, null, 'remote has been deleted'); + // let syncItem = syncItems[i]; + // if (remoteIds.indexOf(syncItem.item_id) < 0) { + // if (syncItem.item_type == Folder.modelType()) { + // localFoldersToDelete.push(syncItem); + // continue; + // } - let ItemClass = BaseItem.itemClass(syncItem.item_type); - await ItemClass.delete(syncItem.item_id, { trackDeleted: false }); - } - } - } + // this.logSyncOperation('deleteLocal', { id: syncItem.item_id }, null, 'remote has been deleted'); - if (!this.cancelling()) { - for (let i = 0; i < localFoldersToDelete.length; i++) { - const syncItem = localFoldersToDelete[i]; - const noteIds = await Folder.noteIds(syncItem.item_id); - if (noteIds.length) { // CONFLICT - await Folder.markNotesAsConflict(syncItem.item_id); - } - await Folder.delete(syncItem.item_id, { deleteChildren: false }); - } - } + // let ItemClass = BaseItem.itemClass(syncItem.item_type); + // await ItemClass.delete(syncItem.item_id, { trackDeleted: false }); + // } + // } + // } - if (!this.cancelling()) { - await BaseItem.deleteOrphanSyncItems(); - } + // if (!this.cancelling()) { + // for (let i = 0; i < localFoldersToDelete.length; i++) { + // const syncItem = localFoldersToDelete[i]; + // const noteIds = await Folder.noteIds(syncItem.item_id); + // if (noteIds.length) { // CONFLICT + // await Folder.markNotesAsConflict(syncItem.item_id); + // } + // await Folder.delete(syncItem.item_id, { deleteChildren: false }); + // } + // } + + // if (!this.cancelling()) { + // await BaseItem.deleteOrphanSyncItems(); + // } } catch (error) { this.logger().error(error); this.progressReport_.errors.push(error); @@ -447,6 +456,8 @@ class Synchronizer { this.progressReport_ = {}; this.dispatch({ type: 'SYNC_COMPLETED' }); + + return outputContext; } }