mirror of
https://github.com/laurent22/joplin.git
synced 2024-12-24 10:27:10 +02:00
All: Improved sync when dealing with many items, in particular when using Nextcloud
This commit is contained in:
parent
c4f19465a6
commit
4df5f668dc
@ -138,7 +138,7 @@ describe("Synchronizer", function() {
|
||||
);
|
||||
|
||||
it(
|
||||
"should update remote item",
|
||||
"should update remote items",
|
||||
asyncTest(async () => {
|
||||
let folder = await Folder.save({ title: "folder1" });
|
||||
let note = await Note.save({ title: "un", parent_id: folder.id });
|
||||
@ -1125,4 +1125,32 @@ describe("Synchronizer", function() {
|
||||
await localItemsSameAsRemote(all, expect);
|
||||
})
|
||||
);
|
||||
|
||||
it(
|
||||
"should update remote items but not pull remote changes",
|
||||
asyncTest(async () => {
|
||||
let folder = await Folder.save({ title: "folder1" });
|
||||
let note = await Note.save({ title: "un", parent_id: folder.id });
|
||||
await synchronizer().start();
|
||||
|
||||
await switchClient(2);
|
||||
|
||||
await synchronizer().start();
|
||||
await Note.save({ title: "deux", parent_id: folder.id });
|
||||
await synchronizer().start();
|
||||
|
||||
await switchClient(1);
|
||||
|
||||
await Note.save({ title: "un UPDATE", id: note.id });
|
||||
await synchronizer().start({ syncSteps: ["update_remote"] });
|
||||
let all = await allItems();
|
||||
expect(all.length).toBe(2);
|
||||
|
||||
await switchClient(2);
|
||||
|
||||
await synchronizer().start();
|
||||
let note2 = await Note.load(note.id);
|
||||
expect(note2.title).toBe("un UPDATE");
|
||||
})
|
||||
);
|
||||
});
|
||||
|
@ -51,8 +51,8 @@ SyncTargetRegistry.addClass(SyncTargetFilesystem);
|
||||
SyncTargetRegistry.addClass(SyncTargetOneDrive);
|
||||
SyncTargetRegistry.addClass(SyncTargetNextcloud);
|
||||
|
||||
const syncTargetId_ = SyncTargetRegistry.nameToId("nextcloud");
|
||||
//const syncTargetId_ = SyncTargetRegistry.nameToId('memory');
|
||||
// const syncTargetId_ = SyncTargetRegistry.nameToId("nextcloud");
|
||||
const syncTargetId_ = SyncTargetRegistry.nameToId("memory");
|
||||
//const syncTargetId_ = SyncTargetRegistry.nameToId('filesystem');
|
||||
const syncDir = __dirname + "/../tests/sync";
|
||||
|
||||
|
@ -143,7 +143,7 @@ class Application extends BaseApplication {
|
||||
}
|
||||
|
||||
if (["NOTE_UPDATE_ONE", "NOTE_DELETE", "FOLDER_UPDATE_ONE", "FOLDER_DELETE"].indexOf(action.type) >= 0) {
|
||||
if (!await reg.syncTarget().syncStarted()) reg.scheduleSync();
|
||||
if (!await reg.syncTarget().syncStarted()) reg.scheduleSync(5, { syncSteps: ["update_remote", "delete_remote"] });
|
||||
}
|
||||
|
||||
if (["EVENT_NOTE_ALARM_FIELD_CHANGE", "NOTE_DELETE"].indexOf(action.type) >= 0) {
|
||||
|
@ -43,8 +43,9 @@ reg.syncTarget = (syncTargetId = null) => {
|
||||
return target;
|
||||
};
|
||||
|
||||
reg.scheduleSync = async (delay = null) => {
|
||||
reg.scheduleSync = async (delay = null, syncOptions = null) => {
|
||||
if (delay === null) delay = 1000 * 10;
|
||||
if (syncOptions === null) syncOptions = {};
|
||||
|
||||
let promiseResolve = null;
|
||||
const promise = new Promise((resolve, reject) => {
|
||||
@ -58,10 +59,10 @@ reg.scheduleSync = async (delay = null) => {
|
||||
|
||||
reg.logger().info("Scheduling sync operation...");
|
||||
|
||||
if (Setting.value("env") === "dev" && delay !== 0) {
|
||||
reg.logger().info("Schedule sync DISABLED!!!");
|
||||
return;
|
||||
}
|
||||
// if (Setting.value("env") === "dev" && delay !== 0) {
|
||||
// reg.logger().info("Schedule sync DISABLED!!!");
|
||||
// return;
|
||||
// }
|
||||
|
||||
const timeoutCallback = async () => {
|
||||
reg.scheduleSyncId_ = null;
|
||||
@ -93,7 +94,8 @@ reg.scheduleSync = async (delay = null) => {
|
||||
|
||||
try {
|
||||
reg.logger().info("Starting scheduled sync");
|
||||
let newContext = await sync.start({ context: context });
|
||||
const options = Object.assign({}, syncOptions, { context: context });
|
||||
const newContext = await sync.start(options);
|
||||
Setting.setValue(contextKey, JSON.stringify(newContext));
|
||||
} catch (error) {
|
||||
if (error.code == "alreadyStarted") {
|
||||
|
@ -157,6 +157,11 @@ class Synchronizer {
|
||||
return this.cancelling_;
|
||||
}
|
||||
|
||||
// Synchronisation is done in three major steps:
|
||||
//
|
||||
// 1. UPLOAD: Send to the sync target the items that have changed since the last sync.
|
||||
// 2. DELETE_REMOTE: Delete on the sync target, the items that have been deleted locally.
|
||||
// 3. DELTA: Find on the sync target the items that have been modified or deleted and apply the changes locally.
|
||||
async start(options = null) {
|
||||
if (!options) options = {};
|
||||
|
||||
@ -173,6 +178,8 @@ class Synchronizer {
|
||||
|
||||
const lastContext = options.context ? options.context : {};
|
||||
|
||||
const syncSteps = options.syncSteps ? options.syncSteps : ["update_remote", "delete_remote", "delta"];
|
||||
|
||||
const syncTargetId = this.api().syncTargetId();
|
||||
|
||||
this.cancelling_ = false;
|
||||
@ -180,11 +187,6 @@ class Synchronizer {
|
||||
const masterKeysBefore = await MasterKey.count();
|
||||
let hasAutoEnabledEncryption = false;
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// First, find all the items that have been changed since the
|
||||
// last sync and apply the changes to remote.
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
let synchronizationId = time.unixMs().toString();
|
||||
|
||||
let outputContext = Object.assign({}, lastContext);
|
||||
@ -198,383 +200,402 @@ class Synchronizer {
|
||||
this.api().setTempDirName(this.syncDirName_);
|
||||
await this.api().mkdir(this.resourceDirName_);
|
||||
|
||||
let donePaths = [];
|
||||
while (true) {
|
||||
if (this.cancelling()) break;
|
||||
// ========================================================================
|
||||
// 1. UPLOAD
|
||||
// ------------------------------------------------------------------------
|
||||
// First, find all the items that have been changed since the
|
||||
// last sync and apply the changes to remote.
|
||||
// ========================================================================
|
||||
|
||||
let result = await BaseItem.itemsThatNeedSync(syncTargetId);
|
||||
let locals = result.items;
|
||||
|
||||
for (let i = 0; i < locals.length; i++) {
|
||||
if (syncSteps.indexOf("update_remote") >= 0) {
|
||||
let donePaths = [];
|
||||
while (true) {
|
||||
if (this.cancelling()) break;
|
||||
|
||||
let local = locals[i];
|
||||
let ItemClass = BaseItem.itemClass(local);
|
||||
let path = BaseItem.systemPath(local);
|
||||
let result = await BaseItem.itemsThatNeedSync(syncTargetId);
|
||||
let locals = result.items;
|
||||
|
||||
// Safety check to avoid infinite loops.
|
||||
// In fact this error is possible if the item is marked for sync (via sync_time or force_sync) while synchronisation is in
|
||||
// progress. In that case exit anyway to be sure we aren't in a loop and the item will be re-synced next time.
|
||||
if (donePaths.indexOf(path) > 0) throw new Error(sprintf("Processing a path that has already been done: %s. sync_time was not updated?", path));
|
||||
for (let i = 0; i < locals.length; i++) {
|
||||
if (this.cancelling()) break;
|
||||
|
||||
let remote = await this.api().stat(path);
|
||||
let action = null;
|
||||
let updateSyncTimeOnly = true;
|
||||
let reason = "";
|
||||
let remoteContent = null;
|
||||
let local = locals[i];
|
||||
let ItemClass = BaseItem.itemClass(local);
|
||||
let path = BaseItem.systemPath(local);
|
||||
|
||||
if (!remote) {
|
||||
if (!local.sync_time) {
|
||||
action = "createRemote";
|
||||
reason = "remote does not exist, and local is new and has never been synced";
|
||||
} else {
|
||||
// Note or item was modified after having been deleted remotely
|
||||
// "itemConflict" if for all the items except the notes, which are dealt with in a special way
|
||||
action = local.type_ == BaseModel.TYPE_NOTE ? "noteConflict" : "itemConflict";
|
||||
reason = "remote has been deleted, but local has changes";
|
||||
}
|
||||
} else {
|
||||
// Note: in order to know the real updated_time value, we need to load the content. In theory we could
|
||||
// rely on the file timestamp (in remote.updated_time) but in practice it's not accurate enough and
|
||||
// can lead to conflicts (for example when the file timestamp is slightly ahead of it's real
|
||||
// updated_time). updated_time is set and managed by clients so it's always accurate.
|
||||
// Same situation below for updateLocal.
|
||||
//
|
||||
// This is a bit inefficient because if the resulting action is "updateRemote" we don't need the whole
|
||||
// content, but for now that will do since being reliable is the priority.
|
||||
//
|
||||
// TODO: assuming a particular sync target is guaranteed to have accurate timestamps, the driver maybe
|
||||
// could expose this with a accurateTimestamps() method that returns "true". In that case, the test
|
||||
// could be done using the file timestamp and the potentially unecessary content loading could be skipped.
|
||||
// OneDrive does not appear to have accurate timestamps as lastModifiedDateTime would occasionally be
|
||||
// a few seconds ahead of what it was set with setTimestamp()
|
||||
remoteContent = await this.api().get(path);
|
||||
if (!remoteContent) throw new Error("Got metadata for path but could not fetch content: " + path);
|
||||
remoteContent = await BaseItem.unserialize(remoteContent);
|
||||
// Safety check to avoid infinite loops.
|
||||
// In fact this error is possible if the item is marked for sync (via sync_time or force_sync) while synchronisation is in
|
||||
// progress. In that case exit anyway to be sure we aren't in a loop and the item will be re-synced next time.
|
||||
if (donePaths.indexOf(path) > 0) throw new Error(sprintf("Processing a path that has already been done: %s. sync_time was not updated?", path));
|
||||
|
||||
if (remoteContent.updated_time > local.sync_time) {
|
||||
// Since, in this loop, we are only dealing with items that require sync, if the
|
||||
// remote has been modified after the sync time, it means both items have been
|
||||
// modified and so there's a conflict.
|
||||
action = local.type_ == BaseModel.TYPE_NOTE ? "noteConflict" : "itemConflict";
|
||||
reason = "both remote and local have changes";
|
||||
} else {
|
||||
action = "updateRemote";
|
||||
reason = "local has changes";
|
||||
}
|
||||
}
|
||||
let remote = await this.api().stat(path);
|
||||
let action = null;
|
||||
let updateSyncTimeOnly = true;
|
||||
let reason = "";
|
||||
let remoteContent = null;
|
||||
|
||||
this.logSyncOperation(action, local, remote, reason);
|
||||
|
||||
const handleCannotSyncItem = async (syncTargetId, item, cannotSyncReason) => {
|
||||
await ItemClass.saveSyncDisabled(syncTargetId, item, cannotSyncReason);
|
||||
this.dispatch({ type: "SYNC_HAS_DISABLED_SYNC_ITEMS" });
|
||||
};
|
||||
|
||||
if (local.type_ == BaseModel.TYPE_RESOURCE && (action == "createRemote" || action === "updateRemote" || (action == "itemConflict" && remote))) {
|
||||
try {
|
||||
const remoteContentPath = this.resourceDirName_ + "/" + local.id;
|
||||
const result = await Resource.fullPathForSyncUpload(local);
|
||||
local = result.resource;
|
||||
const localResourceContentPath = result.path;
|
||||
await this.api().put(remoteContentPath, null, { path: localResourceContentPath, source: "file" });
|
||||
} catch (error) {
|
||||
if (error && ["rejectedByTarget", "fileNotFound"].indexOf(error.code) >= 0) {
|
||||
await handleCannotSyncItem(syncTargetId, local, error.message);
|
||||
action = null;
|
||||
if (!remote) {
|
||||
if (!local.sync_time) {
|
||||
action = "createRemote";
|
||||
reason = "remote does not exist, and local is new and has never been synced";
|
||||
} else {
|
||||
throw error;
|
||||
// Note or item was modified after having been deleted remotely
|
||||
// "itemConflict" if for all the items except the notes, which are dealt with in a special way
|
||||
action = local.type_ == BaseModel.TYPE_NOTE ? "noteConflict" : "itemConflict";
|
||||
reason = "remote has been deleted, but local has changes";
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Note: in order to know the real updated_time value, we need to load the content. In theory we could
|
||||
// rely on the file timestamp (in remote.updated_time) but in practice it's not accurate enough and
|
||||
// can lead to conflicts (for example when the file timestamp is slightly ahead of it's real
|
||||
// updated_time). updated_time is set and managed by clients so it's always accurate.
|
||||
// Same situation below for updateLocal.
|
||||
//
|
||||
// This is a bit inefficient because if the resulting action is "updateRemote" we don't need the whole
|
||||
// content, but for now that will do since being reliable is the priority.
|
||||
//
|
||||
// TODO: assuming a particular sync target is guaranteed to have accurate timestamps, the driver maybe
|
||||
// could expose this with a accurateTimestamps() method that returns "true". In that case, the test
|
||||
// could be done using the file timestamp and the potentially unecessary content loading could be skipped.
|
||||
// OneDrive does not appear to have accurate timestamps as lastModifiedDateTime would occasionally be
|
||||
// a few seconds ahead of what it was set with setTimestamp()
|
||||
remoteContent = await this.api().get(path);
|
||||
if (!remoteContent) throw new Error("Got metadata for path but could not fetch content: " + path);
|
||||
remoteContent = await BaseItem.unserialize(remoteContent);
|
||||
|
||||
if (action == "createRemote" || action == "updateRemote") {
|
||||
let canSync = true;
|
||||
try {
|
||||
if (this.testingHooks_.indexOf("rejectedByTarget") >= 0) throw new JoplinError("Testing rejectedByTarget", "rejectedByTarget");
|
||||
const content = await ItemClass.serializeForSync(local);
|
||||
await this.api().put(path, content);
|
||||
} catch (error) {
|
||||
if (error && error.code === "rejectedByTarget") {
|
||||
await handleCannotSyncItem(syncTargetId, local, error.message);
|
||||
canSync = false;
|
||||
if (remoteContent.updated_time > local.sync_time) {
|
||||
// Since, in this loop, we are only dealing with items that require sync, if the
|
||||
// remote has been modified after the sync time, it means both items have been
|
||||
// modified and so there's a conflict.
|
||||
action = local.type_ == BaseModel.TYPE_NOTE ? "noteConflict" : "itemConflict";
|
||||
reason = "both remote and local have changes";
|
||||
} else {
|
||||
throw error;
|
||||
action = "updateRemote";
|
||||
reason = "local has changes";
|
||||
}
|
||||
}
|
||||
|
||||
// Note: Currently, we set sync_time to update_time, which should work fine given that the resolution is the millisecond.
|
||||
// In theory though, this could happen:
|
||||
//
|
||||
// 1. t0: Editor: Note is modified
|
||||
// 2. t0: Sync: Found that note was modified so start uploading it
|
||||
// 3. t0: Editor: Note is modified again
|
||||
// 4. t1: Sync: Note has finished uploading, set sync_time to t0
|
||||
//
|
||||
// Later any attempt to sync will not detect that note was modified in (3) (within the same millisecond as it was being uploaded)
|
||||
// because sync_time will be t0 too.
|
||||
//
|
||||
// The solution would be to use something like an etag (a simple counter incremented on every change) to make sure each
|
||||
// change is uniquely identified. Leaving it like this for now.
|
||||
this.logSyncOperation(action, local, remote, reason);
|
||||
|
||||
if (canSync) {
|
||||
// 2018-01-21: Setting timestamp is not needed because the delta() logic doesn't rely
|
||||
// on it (instead it uses a more reliable `context` object) and the itemsThatNeedSync loop
|
||||
// above also doesn't use it because it fetches the whole remote object and read the
|
||||
// more reliable 'updated_time' property. Basically remote.updated_time is deprecated.
|
||||
const handleCannotSyncItem = async (syncTargetId, item, cannotSyncReason) => {
|
||||
await ItemClass.saveSyncDisabled(syncTargetId, item, cannotSyncReason);
|
||||
this.dispatch({ type: "SYNC_HAS_DISABLED_SYNC_ITEMS" });
|
||||
};
|
||||
|
||||
// await this.api().setTimestamp(path, local.updated_time);
|
||||
await ItemClass.saveSyncTime(syncTargetId, local, local.updated_time);
|
||||
}
|
||||
} else if (action == "itemConflict") {
|
||||
// ------------------------------------------------------------------------------
|
||||
// For non-note conflicts, we take the remote version (i.e. the version that was
|
||||
// synced first) and overwrite the local content.
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
if (remote) {
|
||||
local = remoteContent;
|
||||
|
||||
const syncTimeQueries = BaseItem.updateSyncTimeQueries(syncTargetId, local, time.unixMs());
|
||||
await ItemClass.save(local, { autoTimestamp: false, nextQueries: syncTimeQueries });
|
||||
} else {
|
||||
await ItemClass.delete(local.id);
|
||||
}
|
||||
} else if (action == "noteConflict") {
|
||||
// ------------------------------------------------------------------------------
|
||||
// First find out if the conflict matters. For example, if the conflict is on the title or body
|
||||
// we want to preserve all the changes. If it's on todo_completed it doesn't really matter
|
||||
// so in this case we just take the remote content.
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
let mustHandleConflict = true;
|
||||
if (remoteContent) {
|
||||
mustHandleConflict = Note.mustHandleConflict(local, remoteContent);
|
||||
if (local.type_ == BaseModel.TYPE_RESOURCE && (action == "createRemote" || action === "updateRemote" || (action == "itemConflict" && remote))) {
|
||||
try {
|
||||
const remoteContentPath = this.resourceDirName_ + "/" + local.id;
|
||||
const result = await Resource.fullPathForSyncUpload(local);
|
||||
local = result.resource;
|
||||
const localResourceContentPath = result.path;
|
||||
await this.api().put(remoteContentPath, null, { path: localResourceContentPath, source: "file" });
|
||||
} catch (error) {
|
||||
if (error && ["rejectedByTarget", "fileNotFound"].indexOf(error.code) >= 0) {
|
||||
await handleCannotSyncItem(syncTargetId, local, error.message);
|
||||
action = null;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
// Create a duplicate of local note into Conflicts folder
|
||||
// (to preserve the user's changes)
|
||||
// ------------------------------------------------------------------------------
|
||||
if (action == "createRemote" || action == "updateRemote") {
|
||||
let canSync = true;
|
||||
try {
|
||||
if (this.testingHooks_.indexOf("rejectedByTarget") >= 0) throw new JoplinError("Testing rejectedByTarget", "rejectedByTarget");
|
||||
const content = await ItemClass.serializeForSync(local);
|
||||
await this.api().put(path, content);
|
||||
} catch (error) {
|
||||
if (error && error.code === "rejectedByTarget") {
|
||||
await handleCannotSyncItem(syncTargetId, local, error.message);
|
||||
canSync = false;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (mustHandleConflict) {
|
||||
let conflictedNote = Object.assign({}, local);
|
||||
delete conflictedNote.id;
|
||||
conflictedNote.is_conflict = 1;
|
||||
await Note.save(conflictedNote, { autoTimestamp: false });
|
||||
// Note: Currently, we set sync_time to update_time, which should work fine given that the resolution is the millisecond.
|
||||
// In theory though, this could happen:
|
||||
//
|
||||
// 1. t0: Editor: Note is modified
|
||||
// 2. t0: Sync: Found that note was modified so start uploading it
|
||||
// 3. t0: Editor: Note is modified again
|
||||
// 4. t1: Sync: Note has finished uploading, set sync_time to t0
|
||||
//
|
||||
// Later any attempt to sync will not detect that note was modified in (3) (within the same millisecond as it was being uploaded)
|
||||
// because sync_time will be t0 too.
|
||||
//
|
||||
// The solution would be to use something like an etag (a simple counter incremented on every change) to make sure each
|
||||
// change is uniquely identified. Leaving it like this for now.
|
||||
|
||||
if (canSync) {
|
||||
// 2018-01-21: Setting timestamp is not needed because the delta() logic doesn't rely
|
||||
// on it (instead it uses a more reliable `context` object) and the itemsThatNeedSync loop
|
||||
// above also doesn't use it because it fetches the whole remote object and read the
|
||||
// more reliable 'updated_time' property. Basically remote.updated_time is deprecated.
|
||||
|
||||
// await this.api().setTimestamp(path, local.updated_time);
|
||||
await ItemClass.saveSyncTime(syncTargetId, local, local.updated_time);
|
||||
}
|
||||
} else if (action == "itemConflict") {
|
||||
// ------------------------------------------------------------------------------
|
||||
// For non-note conflicts, we take the remote version (i.e. the version that was
|
||||
// synced first) and overwrite the local content.
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
if (remote) {
|
||||
local = remoteContent;
|
||||
|
||||
const syncTimeQueries = BaseItem.updateSyncTimeQueries(syncTargetId, local, time.unixMs());
|
||||
await ItemClass.save(local, { autoTimestamp: false, nextQueries: syncTimeQueries });
|
||||
} else {
|
||||
await ItemClass.delete(local.id);
|
||||
}
|
||||
} else if (action == "noteConflict") {
|
||||
// ------------------------------------------------------------------------------
|
||||
// First find out if the conflict matters. For example, if the conflict is on the title or body
|
||||
// we want to preserve all the changes. If it's on todo_completed it doesn't really matter
|
||||
// so in this case we just take the remote content.
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
let mustHandleConflict = true;
|
||||
if (remoteContent) {
|
||||
mustHandleConflict = Note.mustHandleConflict(local, remoteContent);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
// Create a duplicate of local note into Conflicts folder
|
||||
// (to preserve the user's changes)
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
if (mustHandleConflict) {
|
||||
let conflictedNote = Object.assign({}, local);
|
||||
delete conflictedNote.id;
|
||||
conflictedNote.is_conflict = 1;
|
||||
await Note.save(conflictedNote, { autoTimestamp: false });
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
// Either copy the remote content to local or, if the remote content has
|
||||
// been deleted, delete the local content.
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
if (remote) {
|
||||
local = remoteContent;
|
||||
const syncTimeQueries = BaseItem.updateSyncTimeQueries(syncTargetId, local, time.unixMs());
|
||||
await ItemClass.save(local, { autoTimestamp: false, nextQueries: syncTimeQueries });
|
||||
|
||||
if (!!local.encryption_applied) this.dispatch({ type: "SYNC_GOT_ENCRYPTED_ITEM" });
|
||||
} else {
|
||||
// Remote no longer exists (note deleted) so delete local one too
|
||||
await ItemClass.delete(local.id);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
// Either copy the remote content to local or, if the remote content has
|
||||
// been deleted, delete the local content.
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
if (remote) {
|
||||
local = remoteContent;
|
||||
const syncTimeQueries = BaseItem.updateSyncTimeQueries(syncTargetId, local, time.unixMs());
|
||||
await ItemClass.save(local, { autoTimestamp: false, nextQueries: syncTimeQueries });
|
||||
|
||||
if (!!local.encryption_applied) this.dispatch({ type: "SYNC_GOT_ENCRYPTED_ITEM" });
|
||||
} else {
|
||||
// Remote no longer exists (note deleted) so delete local one too
|
||||
await ItemClass.delete(local.id);
|
||||
}
|
||||
donePaths.push(path);
|
||||
}
|
||||
|
||||
donePaths.push(path);
|
||||
if (!result.hasMore) break;
|
||||
}
|
||||
} // UPLOAD STEP
|
||||
|
||||
if (!result.hasMore) break;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// 2. DELETE_REMOTE
|
||||
// ------------------------------------------------------------------------
|
||||
// Delete the remote items that have been deleted locally.
|
||||
// ========================================================================
|
||||
|
||||
if (syncSteps.indexOf("delete_remote") >= 0) {
|
||||
let deletedItems = await BaseItem.deletedItems(syncTargetId);
|
||||
for (let i = 0; i < deletedItems.length; i++) {
|
||||
if (this.cancelling()) break;
|
||||
|
||||
let item = deletedItems[i];
|
||||
let path = BaseItem.systemPath(item.item_id);
|
||||
this.logSyncOperation("deleteRemote", null, { id: item.item_id }, "local has been deleted");
|
||||
await this.api().delete(path);
|
||||
await BaseItem.remoteDeletedItem(syncTargetId, item.item_id);
|
||||
}
|
||||
} // DELETE_REMOTE STEP
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
let deletedItems = await BaseItem.deletedItems(syncTargetId);
|
||||
for (let i = 0; i < deletedItems.length; i++) {
|
||||
if (this.cancelling()) break;
|
||||
|
||||
let item = deletedItems[i];
|
||||
let path = BaseItem.systemPath(item.item_id);
|
||||
this.logSyncOperation("deleteRemote", null, { id: item.item_id }, "local has been deleted");
|
||||
await this.api().delete(path);
|
||||
await BaseItem.remoteDeletedItem(syncTargetId, item.item_id);
|
||||
}
|
||||
|
||||
// 3. DELTA
|
||||
// ------------------------------------------------------------------------
|
||||
// Loop through all the remote items, find those that
|
||||
// have been updated, and apply the changes to local.
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
// At this point all the local items that have changed have been pushed to remote
|
||||
// or handled as conflicts, so no conflict is possible after this.
|
||||
if (syncSteps.indexOf("delta") >= 0) {
|
||||
// At this point all the local items that have changed have been pushed to remote
|
||||
// or handled as conflicts, so no conflict is possible after this.
|
||||
|
||||
let context = null;
|
||||
let newDeltaContext = null;
|
||||
let localFoldersToDelete = [];
|
||||
let hasCancelled = false;
|
||||
if (lastContext.delta) context = lastContext.delta;
|
||||
let context = null;
|
||||
let newDeltaContext = null;
|
||||
let localFoldersToDelete = [];
|
||||
let hasCancelled = false;
|
||||
if (lastContext.delta) context = lastContext.delta;
|
||||
|
||||
while (true) {
|
||||
if (this.cancelling() || hasCancelled) break;
|
||||
while (true) {
|
||||
if (this.cancelling() || hasCancelled) break;
|
||||
|
||||
let listResult = await this.api().delta("", {
|
||||
context: context,
|
||||
let listResult = await this.api().delta("", {
|
||||
context: context,
|
||||
|
||||
// allItemIdsHandler() provides a way for drivers that don't have a delta API to
|
||||
// still provide delta functionality by comparing the items they have to the items
|
||||
// the client has. Very inefficient but that's the only possible workaround.
|
||||
// It's a function so that it is only called if the driver needs these IDs. For
|
||||
// drivers with a delta functionality it's a noop.
|
||||
allItemIdsHandler: async () => {
|
||||
return BaseItem.syncedItemIds(syncTargetId);
|
||||
},
|
||||
});
|
||||
// allItemIdsHandler() provides a way for drivers that don't have a delta API to
|
||||
// still provide delta functionality by comparing the items they have to the items
|
||||
// the client has. Very inefficient but that's the only possible workaround.
|
||||
// It's a function so that it is only called if the driver needs these IDs. For
|
||||
// drivers with a delta functionality it's a noop.
|
||||
allItemIdsHandler: async () => {
|
||||
return BaseItem.syncedItemIds(syncTargetId);
|
||||
},
|
||||
});
|
||||
|
||||
let remotes = listResult.items;
|
||||
let remotes = listResult.items;
|
||||
|
||||
this.logSyncOperation("fetchingTotal", null, null, "Fetching delta items from sync target", remotes.length);
|
||||
this.logSyncOperation("fetchingTotal", null, null, "Fetching delta items from sync target", remotes.length);
|
||||
|
||||
for (let i = 0; i < remotes.length; i++) {
|
||||
if (this.cancelling() || this.testingHooks_.indexOf("cancelDeltaLoop2") >= 0) {
|
||||
hasCancelled = true;
|
||||
break;
|
||||
}
|
||||
|
||||
this.logSyncOperation("fetchingProcessed", null, null, "Processing fetched item");
|
||||
|
||||
let remote = remotes[i];
|
||||
if (!BaseItem.isSystemPath(remote.path)) continue; // The delta API might return things like the .sync, .resource or the root folder
|
||||
|
||||
const loadContent = async () => {
|
||||
let content = await this.api().get(path);
|
||||
if (!content) return null;
|
||||
return await BaseItem.unserialize(content);
|
||||
};
|
||||
|
||||
let path = remote.path;
|
||||
let action = null;
|
||||
let reason = "";
|
||||
let local = await BaseItem.loadItemByPath(path);
|
||||
let ItemClass = null;
|
||||
let content = null;
|
||||
if (!local) {
|
||||
if (remote.isDeleted !== true) {
|
||||
action = "createLocal";
|
||||
reason = "remote exists but local does not";
|
||||
content = await loadContent();
|
||||
ItemClass = content ? BaseItem.itemClass(content) : null;
|
||||
for (let i = 0; i < remotes.length; i++) {
|
||||
if (this.cancelling() || this.testingHooks_.indexOf("cancelDeltaLoop2") >= 0) {
|
||||
hasCancelled = true;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
ItemClass = BaseItem.itemClass(local);
|
||||
local = ItemClass.filter(local);
|
||||
if (remote.isDeleted) {
|
||||
action = "deleteLocal";
|
||||
reason = "remote has been deleted";
|
||||
|
||||
this.logSyncOperation("fetchingProcessed", null, null, "Processing fetched item");
|
||||
|
||||
let remote = remotes[i];
|
||||
if (!BaseItem.isSystemPath(remote.path)) continue; // The delta API might return things like the .sync, .resource or the root folder
|
||||
|
||||
const loadContent = async () => {
|
||||
let content = await this.api().get(path);
|
||||
if (!content) return null;
|
||||
return await BaseItem.unserialize(content);
|
||||
};
|
||||
|
||||
let path = remote.path;
|
||||
let action = null;
|
||||
let reason = "";
|
||||
let local = await BaseItem.loadItemByPath(path);
|
||||
let ItemClass = null;
|
||||
let content = null;
|
||||
if (!local) {
|
||||
if (remote.isDeleted !== true) {
|
||||
action = "createLocal";
|
||||
reason = "remote exists but local does not";
|
||||
content = await loadContent();
|
||||
ItemClass = content ? BaseItem.itemClass(content) : null;
|
||||
}
|
||||
} else {
|
||||
content = await loadContent();
|
||||
if (content && content.updated_time > local.updated_time) {
|
||||
action = "updateLocal";
|
||||
reason = "remote is more recent than local";
|
||||
ItemClass = BaseItem.itemClass(local);
|
||||
local = ItemClass.filter(local);
|
||||
if (remote.isDeleted) {
|
||||
action = "deleteLocal";
|
||||
reason = "remote has been deleted";
|
||||
} else {
|
||||
content = await loadContent();
|
||||
if (content && content.updated_time > local.updated_time) {
|
||||
action = "updateLocal";
|
||||
reason = "remote is more recent than local";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!action) continue;
|
||||
|
||||
this.logSyncOperation(action, local, remote, reason);
|
||||
|
||||
if (action == "createLocal" || action == "updateLocal") {
|
||||
if (content === null) {
|
||||
this.logger().warn("Remote has been deleted between now and the delta() call? In that case it will be handled during the next sync: " + path);
|
||||
continue;
|
||||
}
|
||||
content = ItemClass.filter(content);
|
||||
|
||||
// 2017-12-03: This was added because the new user_updated_time and user_created_time properties were added
|
||||
// to the items. However changing the database is not enough since remote items that haven't been synced yet
|
||||
// will not have these properties and, since they are required, it would cause a problem. So this check
|
||||
// if they are present and, if not, set them to a reasonable default.
|
||||
// Let's leave these two lines for 6 months, by which time all the clients should have been synced.
|
||||
if (!content.user_updated_time) content.user_updated_time = content.updated_time;
|
||||
if (!content.user_created_time) content.user_created_time = content.created_time;
|
||||
|
||||
let options = {
|
||||
autoTimestamp: false,
|
||||
nextQueries: BaseItem.updateSyncTimeQueries(syncTargetId, content, time.unixMs()),
|
||||
};
|
||||
if (action == "createLocal") options.isNew = true;
|
||||
if (action == "updateLocal") options.oldItem = local;
|
||||
|
||||
if (content.type_ == BaseModel.TYPE_RESOURCE && action == "createLocal") {
|
||||
let localResourceContentPath = Resource.fullPath(content);
|
||||
let remoteResourceContentPath = this.resourceDirName_ + "/" + content.id;
|
||||
await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" });
|
||||
}
|
||||
|
||||
await ItemClass.save(content, options);
|
||||
|
||||
if (!hasAutoEnabledEncryption && content.type_ === BaseModel.TYPE_MASTER_KEY && !masterKeysBefore) {
|
||||
hasAutoEnabledEncryption = true;
|
||||
this.logger().info("One master key was downloaded and none was previously available: automatically enabling encryption");
|
||||
this.logger().info("Using master key: ", content);
|
||||
await this.encryptionService().enableEncryption(content);
|
||||
await this.encryptionService().loadMasterKeysFromSettings();
|
||||
this.logger().info(
|
||||
"Encryption has been enabled with downloaded master key as active key. However, note that no password was initially supplied. It will need to be provided by user."
|
||||
);
|
||||
}
|
||||
|
||||
if (!!content.encryption_applied) this.dispatch({ type: "SYNC_GOT_ENCRYPTED_ITEM" });
|
||||
} else if (action == "deleteLocal") {
|
||||
if (local.type_ == BaseModel.TYPE_FOLDER) {
|
||||
localFoldersToDelete.push(local);
|
||||
continue;
|
||||
}
|
||||
|
||||
let ItemClass = BaseItem.itemClass(local.type_);
|
||||
await ItemClass.delete(local.id, { trackDeleted: false });
|
||||
}
|
||||
}
|
||||
|
||||
if (!action) continue;
|
||||
|
||||
this.logSyncOperation(action, local, remote, reason);
|
||||
|
||||
if (action == "createLocal" || action == "updateLocal") {
|
||||
if (content === null) {
|
||||
this.logger().warn("Remote has been deleted between now and the delta() call? In that case it will be handled during the next sync: " + path);
|
||||
continue;
|
||||
// If user has cancelled, don't record the new context (2) so that synchronisation
|
||||
// can start again from the previous context (1) next time. It is ok if some items
|
||||
// have been synced between (1) and (2) because the loop above will handle the same
|
||||
// items being synced twice as an update. If the local and remote items are indentical
|
||||
// the update will simply be skipped.
|
||||
if (!hasCancelled) {
|
||||
if (!listResult.hasMore) {
|
||||
newDeltaContext = listResult.context;
|
||||
break;
|
||||
}
|
||||
content = ItemClass.filter(content);
|
||||
|
||||
// 2017-12-03: This was added because the new user_updated_time and user_created_time properties were added
|
||||
// to the items. However changing the database is not enough since remote items that haven't been synced yet
|
||||
// will not have these properties and, since they are required, it would cause a problem. So this check
|
||||
// if they are present and, if not, set them to a reasonable default.
|
||||
// Let's leave these two lines for 6 months, by which time all the clients should have been synced.
|
||||
if (!content.user_updated_time) content.user_updated_time = content.updated_time;
|
||||
if (!content.user_created_time) content.user_created_time = content.created_time;
|
||||
|
||||
let options = {
|
||||
autoTimestamp: false,
|
||||
nextQueries: BaseItem.updateSyncTimeQueries(syncTargetId, content, time.unixMs()),
|
||||
};
|
||||
if (action == "createLocal") options.isNew = true;
|
||||
if (action == "updateLocal") options.oldItem = local;
|
||||
|
||||
if (content.type_ == BaseModel.TYPE_RESOURCE && action == "createLocal") {
|
||||
let localResourceContentPath = Resource.fullPath(content);
|
||||
let remoteResourceContentPath = this.resourceDirName_ + "/" + content.id;
|
||||
await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" });
|
||||
}
|
||||
|
||||
await ItemClass.save(content, options);
|
||||
|
||||
if (!hasAutoEnabledEncryption && content.type_ === BaseModel.TYPE_MASTER_KEY && !masterKeysBefore) {
|
||||
hasAutoEnabledEncryption = true;
|
||||
this.logger().info("One master key was downloaded and none was previously available: automatically enabling encryption");
|
||||
this.logger().info("Using master key: ", content);
|
||||
await this.encryptionService().enableEncryption(content);
|
||||
await this.encryptionService().loadMasterKeysFromSettings();
|
||||
this.logger().info("Encryption has been enabled with downloaded master key as active key. However, note that no password was initially supplied. It will need to be provided by user.");
|
||||
}
|
||||
|
||||
if (!!content.encryption_applied) this.dispatch({ type: "SYNC_GOT_ENCRYPTED_ITEM" });
|
||||
} else if (action == "deleteLocal") {
|
||||
if (local.type_ == BaseModel.TYPE_FOLDER) {
|
||||
localFoldersToDelete.push(local);
|
||||
continue;
|
||||
}
|
||||
|
||||
let ItemClass = BaseItem.itemClass(local.type_);
|
||||
await ItemClass.delete(local.id, { trackDeleted: false });
|
||||
context = listResult.context;
|
||||
}
|
||||
}
|
||||
|
||||
// If user has cancelled, don't record the new context (2) so that synchronisation
|
||||
// can start again from the previous context (1) next time. It is ok if some items
|
||||
// have been synced between (1) and (2) because the loop above will handle the same
|
||||
// items being synced twice as an update. If the local and remote items are indentical
|
||||
// the update will simply be skipped.
|
||||
if (!hasCancelled) {
|
||||
if (!listResult.hasMore) {
|
||||
newDeltaContext = listResult.context;
|
||||
break;
|
||||
outputContext.delta = newDeltaContext ? newDeltaContext : lastContext.delta;
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// Delete the folders that have been collected in the loop above.
|
||||
// Folders are always deleted last, and only if they are empty.
|
||||
// If they are not empty it's considered a conflict since whatever deleted
|
||||
// them should have deleted their content too. In that case, all its notes
|
||||
// are marked as "is_conflict".
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
if (!this.cancelling()) {
|
||||
for (let i = 0; i < localFoldersToDelete.length; i++) {
|
||||
const item = localFoldersToDelete[i];
|
||||
const noteIds = await Folder.noteIds(item.id);
|
||||
if (noteIds.length) {
|
||||
// CONFLICT
|
||||
await Folder.markNotesAsConflict(item.id);
|
||||
}
|
||||
await Folder.delete(item.id, { deleteChildren: false, trackDeleted: false });
|
||||
}
|
||||
context = listResult.context;
|
||||
}
|
||||
}
|
||||
|
||||
outputContext.delta = newDeltaContext ? newDeltaContext : lastContext.delta;
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// Delete the folders that have been collected in the loop above.
|
||||
// Folders are always deleted last, and only if they are empty.
|
||||
// If they are not empty it's considered a conflict since whatever deleted
|
||||
// them should have deleted their content too. In that case, all its notes
|
||||
// are marked as "is_conflict".
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
if (!this.cancelling()) {
|
||||
for (let i = 0; i < localFoldersToDelete.length; i++) {
|
||||
const item = localFoldersToDelete[i];
|
||||
const noteIds = await Folder.noteIds(item.id);
|
||||
if (noteIds.length) {
|
||||
// CONFLICT
|
||||
await Folder.markNotesAsConflict(item.id);
|
||||
}
|
||||
await Folder.delete(item.id, { deleteChildren: false, trackDeleted: false });
|
||||
if (!this.cancelling()) {
|
||||
await BaseItem.deleteOrphanSyncItems();
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.cancelling()) {
|
||||
await BaseItem.deleteOrphanSyncItems();
|
||||
}
|
||||
} // DELTA STEP
|
||||
} catch (error) {
|
||||
if (error && ["cannotEncryptEncrypted", "noActiveMasterKey"].indexOf(error.code) >= 0) {
|
||||
// Only log an info statement for this since this is a common condition that is reported
|
||||
|
@ -79,7 +79,7 @@ const generalMiddleware = store => next => async action => {
|
||||
if (action.type == "NAV_GO") Keyboard.dismiss();
|
||||
|
||||
if (["NOTE_UPDATE_ONE", "NOTE_DELETE", "FOLDER_UPDATE_ONE", "FOLDER_DELETE"].indexOf(action.type) >= 0) {
|
||||
if (!await reg.syncTarget().syncStarted()) reg.scheduleSync();
|
||||
if (!await reg.syncTarget().syncStarted()) reg.scheduleSync(5, { syncSteps: ["update_remote", "delete_remote"] });
|
||||
}
|
||||
|
||||
if (["EVENT_NOTE_ALARM_FIELD_CHANGE", "NOTE_DELETE"].indexOf(action.type) >= 0) {
|
||||
|
@ -75,6 +75,9 @@
|
||||
"js_prettier": {
|
||||
"auto_format_on_save": true,
|
||||
"auto_format_on_save_requires_prettier_config": true,
|
||||
"additional_cli_args": {
|
||||
"--config": "prettier.config.js",
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user