From 02ff02a9d9f957dab91535f7fafbb096714c44a2 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sun, 18 Jun 2017 23:06:10 +0100 Subject: [PATCH] refactored sync class --- CliClient/app/cmd.js | 4 +- CliClient/app/import-enex.js | 10 +- CliClient/tests/synchronizer.js | 277 ++--------- QtClient/JoplinQtClient/cliapplication.cpp | 10 +- QtClient/JoplinQtClient/models/item.cpp | 2 +- QtClient/JoplinQtClient/models/item.h | 2 +- QtClient/JoplinQtClient/settings.cpp | 2 +- QtClient/JoplinQtClient/settings.h | 2 +- ReactNativeClient/src/base-model.js | 4 +- .../src/file-api-driver-local.js | 8 +- .../src/file-api-driver-memory.js | 10 +- ReactNativeClient/src/file-api.js | 31 -- ReactNativeClient/src/models/base-item.js | 50 +- ReactNativeClient/src/models/folder.js | 4 +- ReactNativeClient/src/models/note.js | 4 +- ReactNativeClient/src/synchronizer.js | 466 ++++-------------- ReactNativeClient/src/synchronizer_old.js | 6 +- ReactNativeClient/src/synchronizer_old2.js | 418 ++++++++++++++++ ReactNativeClient/src/time-utils.js | 8 + 19 files changed, 628 insertions(+), 690 deletions(-) create mode 100644 ReactNativeClient/src/synchronizer_old2.js diff --git a/CliClient/app/cmd.js b/CliClient/app/cmd.js index 29f648cd0..251eb2626 100644 --- a/CliClient/app/cmd.js +++ b/CliClient/app/cmd.js @@ -463,9 +463,9 @@ async function createLocalItems() { // } // if (!currentFolder) { -// this.log(Folder.toFriendlyString(item)); +// this.log(Folder.serialize(item)); // } else { -// this.log(Note.toFriendlyString(item)); +// this.log(Note.serialize(item)); // } // }).catch((error) => { // this.log(error); diff --git a/CliClient/app/import-enex.js b/CliClient/app/import-enex.js index c818d91ea..a9457459b 100644 --- a/CliClient/app/import-enex.js +++ b/CliClient/app/import-enex.js @@ -527,7 +527,7 @@ function saveNoteToWebApi(note) { }); } -function noteToFriendlyString_format(propName, propValue) { +function noteserialize_format(propName, propValue) { if (['created_time', 'updated_time'].indexOf(propName) >= 0) { if (!propValue) return ''; propValue = moment.unix(propValue).format('YYYY-MM-DD hh:mm:ss'); @@ -538,7 +538,7 @@ function noteToFriendlyString_format(propName, propValue) { return propValue; } -function noteToFriendlyString(note) { +function noteserialize(note) { let shownKeys = ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time']; let output = []; @@ -548,7 +548,7 @@ function noteToFriendlyString(note) { output.push(''); for (let i = 0; i < shownKeys.length; i++) { let v = note[shownKeys[i]]; - v = noteToFriendlyString_format(shownKeys[i], v); + v = noteserialize_format(shownKeys[i], v); output.push(shownKeys[i] + ': ' + v); } @@ -623,7 +623,7 @@ const baseNoteDir = '/home/laurent/Temp/TestImport'; // }); function saveNoteToDisk(folder, note) { - const noteContent = noteToFriendlyString(note); + const noteContent = noteserialize(note); const notePath = baseNoteDir + '/' + folderFilename(folder) + '/' + noteFilename(note); // console.info('==================================================='); @@ -694,7 +694,7 @@ function importEnex(parentFolder, stream) { saveNoteToDisk(parentFolder, note); - // console.info(noteToFriendlyString(note)); + // console.info(noteserialize(note)); // console.info('========================================================================================================================='); //saveNoteToWebApi(note); diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index f1ae3759e..79adf8ff0 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -6,8 +6,16 @@ import { Note } from 'src/models/note.js'; import { BaseItem } from 'src/models/base-item.js'; import { BaseModel } from 'src/base-model.js'; +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); + // application specific logging, throwing an error, or other logic here +}); + async function localItemsSameAsRemote(locals, expect) { try { + let files = await fileApi().list(); + expect(locals.length).toBe(files.length); + for (let i = 0; i < locals.length; i++) { let dbItem = locals[i]; let path = BaseItem.systemPath(dbItem); @@ -19,10 +27,10 @@ async function localItemsSameAsRemote(locals, expect) { // console.info('======================='); expect(!!remote).toBe(true); - expect(remote.updatedTime).toBe(dbItem.updated_time); + expect(remote.updated_time).toBe(dbItem.updated_time); let remoteContent = await fileApi().get(path); - remoteContent = dbItem.type_ == BaseModel.ITEM_TYPE_NOTE ? Note.fromFriendlyString(remoteContent) : Folder.fromFriendlyString(remoteContent); + remoteContent = dbItem.type_ == BaseModel.ITEM_TYPE_NOTE ? Note.unserialize(remoteContent) : Folder.unserialize(remoteContent); expect(remoteContent.title).toBe(dbItem.title); } } catch (error) { @@ -39,94 +47,72 @@ describe('Synchronizer', function() { done(); }); - // it('should create remote items', async (done) => { - // let folder = await Folder.save({ title: "folder1" }); - // await Note.save({ title: "un", parent_id: folder.id }); + it('should create remote items', async (done) => { + let folder = await Folder.save({ title: "folder1" }); + await Note.save({ title: "un", parent_id: folder.id }); - // let all = await Folder.all(true); + let all = await Folder.all(true); - // await synchronizer().start(); + await synchronizer().start(); - // await localItemsSameAsRemote(all, expect); + await localItemsSameAsRemote(all, expect); - // done(); - // }); + done(); + }); - // it('should update remote item', async (done) => { - // let folder = await Folder.save({ title: "folder1" }); - // let note = await Note.save({ title: "un", parent_id: folder.id }); + it('should update remote item', async (done) => { + let folder = await Folder.save({ title: "folder1" }); + let note = await Note.save({ title: "un", parent_id: folder.id }); + await synchronizer().start(); - // await sleep(1); + await sleep(1); - // await Note.save({ title: "un UPDATE", id: note.id }); + await Note.save({ title: "un UPDATE", id: note.id }); - // let all = await Folder.all(true); - // await synchronizer().start(); + let all = await Folder.all(true); + await synchronizer().start(); - // await localItemsSameAsRemote(all, expect); + await localItemsSameAsRemote(all, expect); - // done(); - // }); + done(); + }); - // it('should create local items', async (done) => { - // let folder = await Folder.save({ title: "folder1" }); - // await Note.save({ title: "un", parent_id: folder.id }); - // await synchronizer().start(); - // await clearDatabase(); - // await synchronizer().start(); + it('should create local items', async (done) => { + let folder = await Folder.save({ title: "folder1" }); + await Note.save({ title: "un", parent_id: folder.id }); + await synchronizer().start(); - // let all = await Folder.all(true); - // await localItemsSameAsRemote(all, expect); + switchClient(2); - // done(); - // }); + await synchronizer().start(); - // it('should create same items on client 2', async (done) => { - // let folder = await Folder.save({ title: "folder1" }); - // let note = await Note.save({ title: "un", parent_id: folder.id }); - // await synchronizer().start(); + let all = await Folder.all(true); + await localItemsSameAsRemote(all, expect); - // await sleep(1); - - // switchClient(2); - - // await synchronizer().start(); - - // let folder2 = await Folder.load(folder.id); - // let note2 = await Note.load(note.id); - - // expect(!!folder2).toBe(true); - // expect(!!note2).toBe(true); - - // expect(folder.title).toBe(folder.title); - // expect(folder.updated_time).toBe(folder.updated_time); - - // expect(note.title).toBe(note.title); - // expect(note.updated_time).toBe(note.updated_time); - // expect(note.body).toBe(note.body); - - // done(); - // }); + done(); + }); it('should update local items', async (done) => { let folder1 = await Folder.save({ title: "folder1" }); let note1 = await Note.save({ title: "un", parent_id: folder1.id }); await synchronizer().start(); - await sleep(1); - switchClient(2); await synchronizer().start(); + await sleep(1); + let note2 = await Note.load(note1.id); note2.title = "Updated on client 2"; await Note.save(note2); - let all = await Folder.all(true); + note2 = await Note.load(note2.id); await synchronizer().start(); + let files = await fileApi().list(); + switchClient(1); await synchronizer().start(); @@ -140,177 +126,4 @@ describe('Synchronizer', function() { done(); }); -}); - -// // Note: set 1 matches set 1 of createRemoteItems() -// function createLocalItems(id, updatedTime, lastSyncTime) { -// let output = []; -// if (id === 1) { -// output.push({ path: 'test', isDir: true, updatedTime: updatedTime, lastSyncTime: lastSyncTime }); -// output.push({ path: 'test/un', updatedTime: updatedTime, lastSyncTime: lastSyncTime }); -// } else { -// throw new Error('Invalid ID'); -// } -// return output; -// } - -// function createRemoteItems(id = 1, updatedTime = null) { -// if (!updatedTime) updatedTime = time.unix(); - -// if (id === 1) { -// return fileApi().format() -// .then(() => fileApi().mkdir('test')) -// .then(() => fileApi().put('test/un', 'abcd')) -// .then(() => fileApi().list('', true)) -// .then((items) => { -// for (let i = 0; i < items.length; i++) { -// items[i].updatedTime = updatedTime; -// } -// return items; -// }); -// } else { -// throw new Error('Invalid ID'); -// } -// } - -// describe('Synchronizer syncActions', function() { - -// beforeEach(function(done) { -// setupDatabaseAndSynchronizer(done); -// }); - -// it('should create remote items', function() { -// let localItems = createLocalItems(1, time.unix(), 0); -// let remoteItems = []; - -// let actions = synchronizer().syncActions(localItems, remoteItems, []); - -// expect(actions.length).toBe(2); -// for (let i = 0; i < actions.length; i++) { -// expect(actions[i].type).toBe('create'); -// expect(actions[i].dest).toBe('remote'); -// } -// }); - -// it('should update remote items', function(done) { -// createRemoteItems(1).then((remoteItems) => { -// let lastSyncTime = time.unix() + 1000; -// let localItems = createLocalItems(1, lastSyncTime + 1000, lastSyncTime); -// let actions = synchronizer().syncActions(localItems, remoteItems, []); - -// expect(actions.length).toBe(2); -// for (let i = 0; i < actions.length; i++) { -// expect(actions[i].type).toBe('update'); -// expect(actions[i].dest).toBe('remote'); -// } - -// done(); -// }); -// }); - -// it('should detect conflict', function(done) { -// // Simulate this scenario: -// // - Client 1 create items -// // - Client 1 sync -// // - Client 2 sync -// // - Client 2 change items -// // - Client 2 sync -// // - Client 1 change items -// // - Client 1 sync -// // => Conflict - -// createRemoteItems(1).then((remoteItems) => { -// let localItems = createLocalItems(1, time.unix() + 1000, time.unix() - 1000); -// let actions = synchronizer().syncActions(localItems, remoteItems, []); - -// expect(actions.length).toBe(2); -// for (let i = 0; i < actions.length; i++) { -// expect(actions[i].type).toBe('conflict'); -// } - -// done(); -// }); -// }); - - -// it('should create local file', function(done) { -// createRemoteItems(1).then((remoteItems) => { -// let localItems = []; -// let actions = synchronizer().syncActions(localItems, remoteItems, []); - -// expect(actions.length).toBe(2); -// for (let i = 0; i < actions.length; i++) { -// expect(actions[i].type).toBe('create'); -// expect(actions[i].dest).toBe('local'); -// } - -// done(); -// }); -// }); - -// it('should delete remote files', function(done) { -// createRemoteItems(1).then((remoteItems) => { -// let localItems = createLocalItems(1, time.unix(), time.unix()); -// let deletedItemPaths = [localItems[0].path, localItems[1].path]; -// let actions = synchronizer().syncActions([], remoteItems, deletedItemPaths); - -// expect(actions.length).toBe(2); -// for (let i = 0; i < actions.length; i++) { -// expect(actions[i].type).toBe('delete'); -// expect(actions[i].dest).toBe('remote'); -// } - -// done(); -// }); -// }); - -// it('should delete local files', function(done) { -// let lastSyncTime = time.unix(); -// createRemoteItems(1, lastSyncTime - 1000).then((remoteItems) => { -// let localItems = createLocalItems(1, lastSyncTime - 1000, lastSyncTime); -// let actions = synchronizer().syncActions(localItems, [], []); - -// expect(actions.length).toBe(2); -// for (let i = 0; i < actions.length; i++) { -// expect(actions[i].type).toBe('delete'); -// expect(actions[i].dest).toBe('local'); -// } - -// done(); -// }); -// }); - -// it('should update local files', function(done) { -// let lastSyncTime = time.unix(); -// createRemoteItems(1, lastSyncTime + 1000).then((remoteItems) => { -// let localItems = createLocalItems(1, lastSyncTime - 1000, lastSyncTime); -// let actions = synchronizer().syncActions(localItems, remoteItems, []); - -// expect(actions.length).toBe(2); -// for (let i = 0; i < actions.length; i++) { -// expect(actions[i].type).toBe('update'); -// expect(actions[i].dest).toBe('local'); -// } - -// done(); -// }); -// }); - -// }); - -// // describe('Synchronizer start', function() { - -// // beforeEach(function(done) { -// // setupDatabaseAndSynchronizer(done); -// // }); - -// // it('should create remote items', function(done) { -// // createFoldersAndNotes().then(() => { -// // return synchronizer().start(); -// // } -// // }).then(() => { -// // done(); -// // }); - -// // }); - +}); \ No newline at end of file diff --git a/QtClient/JoplinQtClient/cliapplication.cpp b/QtClient/JoplinQtClient/cliapplication.cpp index 76a78f42a..01ab85b01 100755 --- a/QtClient/JoplinQtClient/cliapplication.cpp +++ b/QtClient/JoplinQtClient/cliapplication.cpp @@ -135,13 +135,13 @@ void CliApplication::saveNoteIfFileChanged(Note& note, const QDateTime& original // if (propKey.isEmpty()) { // QStringList propKeys = settings.allKeys(); // for (int i = 0; i < propKeys.size(); i++) { -// qStdout() << settings.keyValueToFriendlyString(propKeys[i]) << endl; +// qStdout() << settings.keyValueserialize(propKeys[i]) << endl; // } // return 0; // } // if (propValue.isEmpty()) { -// qStdout() << settings.keyValueToFriendlyString(propKey) << endl; +// qStdout() << settings.keyValueserialize(propKey) << endl; // return 0; // } @@ -386,7 +386,7 @@ int CliApplication::exec() { QString noteFilePath = QString("%1/%2.txt").arg(paths::noteDraftsDir()).arg(note.idString()); - if (!filePutContents(noteFilePath, note.toFriendlyString())) { + if (!filePutContents(noteFilePath, note.serialize())) { qStderr() << QString("Cannot open %1 for writing").arg(noteFilePath) << endl; return 1; } @@ -431,13 +431,13 @@ int CliApplication::exec() { if (propKey.isEmpty()) { QStringList propKeys = settings.allKeys(); for (int i = 0; i < propKeys.size(); i++) { - qStdout() << settings.keyValueToFriendlyString(propKeys[i]) << endl; + qStdout() << settings.keyValueserialize(propKeys[i]) << endl; } return 0; } if (propValue.isEmpty()) { - qStdout() << settings.keyValueToFriendlyString(propKey) << endl; + qStdout() << settings.keyValueserialize(propKey) << endl; return 0; } diff --git a/QtClient/JoplinQtClient/models/item.cpp b/QtClient/JoplinQtClient/models/item.cpp index e15b6df7b..1e58ed052 100755 --- a/QtClient/JoplinQtClient/models/item.cpp +++ b/QtClient/JoplinQtClient/models/item.cpp @@ -5,7 +5,7 @@ namespace jop { Item::Item() {} -QString Item::toFriendlyString() const { +QString Item::serialize() const { QStringList shownKeys; shownKeys << "author" << "longitude" << "latitude" << "is_todo" << "todo_due" << "todo_completed"; diff --git a/QtClient/JoplinQtClient/models/item.h b/QtClient/JoplinQtClient/models/item.h index 9c93d9fc0..8eb262d8f 100755 --- a/QtClient/JoplinQtClient/models/item.h +++ b/QtClient/JoplinQtClient/models/item.h @@ -14,7 +14,7 @@ class Item : public BaseModel { public: Item(); - QString toFriendlyString() const; + QString serialize() const; void patchFriendlyString(const QString& patch); }; diff --git a/QtClient/JoplinQtClient/settings.cpp b/QtClient/JoplinQtClient/settings.cpp index a6e52c893..280168225 100755 --- a/QtClient/JoplinQtClient/settings.cpp +++ b/QtClient/JoplinQtClient/settings.cpp @@ -35,6 +35,6 @@ int Settings::valueInt(const QString &name, int defaultValue) { return value(name, defaultValue).toInt(); } -QString Settings::keyValueToFriendlyString(const QString& key) const { +QString Settings::keyValueserialize(const QString& key) const { return QString("%1 = %2").arg(key).arg(value(key).toString()); } \ No newline at end of file diff --git a/QtClient/JoplinQtClient/settings.h b/QtClient/JoplinQtClient/settings.h index b4216d137..4a8e75e54 100755 --- a/QtClient/JoplinQtClient/settings.h +++ b/QtClient/JoplinQtClient/settings.h @@ -15,7 +15,7 @@ public: Settings(); static void initialize(); - QString keyValueToFriendlyString(const QString& key) const; + QString keyValueserialize(const QString& key) const; public slots: diff --git a/ReactNativeClient/src/base-model.js b/ReactNativeClient/src/base-model.js index 2190859a8..d78184c50 100644 --- a/ReactNativeClient/src/base-model.js +++ b/ReactNativeClient/src/base-model.js @@ -161,7 +161,7 @@ class BaseModel { let itemId = o.id; if (options.autoTimestamp && this.hasField('updated_time')) { - o.updated_time = time.unix(); + o.updated_time = time.unixMs(); } if (options.isNew) { @@ -171,7 +171,7 @@ class BaseModel { } if (!o.created_time && this.hasField('created_time')) { - o.created_time = time.unix(); + o.created_time = time.unixMs(); } query = Database.insertQuery(this.tableName(), o); diff --git a/ReactNativeClient/src/file-api-driver-local.js b/ReactNativeClient/src/file-api-driver-local.js index 077463817..e0e268a21 100644 --- a/ReactNativeClient/src/file-api-driver-local.js +++ b/ReactNativeClient/src/file-api-driver-local.js @@ -32,10 +32,10 @@ class FileApiDriverLocal { metadataFromStats_(path, stats) { return { path: path, - createdTime: this.statTimeToUnixTimestamp_(stats.birthtime), - updatedTime: this.statTimeToUnixTimestamp_(stats.mtime), - createdTimeOrig: stats.birthtime, - updatedTimeOrig: stats.mtime, + created_time: this.statTimeToUnixTimestamp_(stats.birthtime), + updated_time: this.statTimeToUnixTimestamp_(stats.mtime), + created_time_orig: stats.birthtime, + updated_time_orig: stats.mtime, isDir: stats.isDirectory(), }; } diff --git a/ReactNativeClient/src/file-api-driver-memory.js b/ReactNativeClient/src/file-api-driver-memory.js index aed1bf5e7..4a6b66f77 100644 --- a/ReactNativeClient/src/file-api-driver-memory.js +++ b/ReactNativeClient/src/file-api-driver-memory.js @@ -23,12 +23,12 @@ class FileApiDriverMemory { } newItem(path, isDir = false) { - let now = time.unix(); + let now = time.unixMs(); return { path: path, isDir: isDir, - updatedTime: now, - createdTime: now, + updated_time: now, // In milliseconds!! + created_time: now, // In milliseconds!! content: '', }; } @@ -41,7 +41,7 @@ class FileApiDriverMemory { setTimestamp(path, timestamp) { let item = this.itemByPath(path); if (!item) return Promise.reject(new Error('File not found: ' + path)); - item.updatedTime = timestamp; + item.updated_time = timestamp; return Promise.resolve(); } @@ -85,7 +85,7 @@ class FileApiDriverMemory { this.items_.push(item); } else { this.items_[index].content = content; - this.items_[index].updatedTime = time.unix(); + this.items_[index].updated_time = time.unix(); } return Promise.resolve(); } diff --git a/ReactNativeClient/src/file-api.js b/ReactNativeClient/src/file-api.js index 359d504a1..8ed500715 100644 --- a/ReactNativeClient/src/file-api.js +++ b/ReactNativeClient/src/file-api.js @@ -41,34 +41,6 @@ class FileApi { return this.driver_.list(this.baseDir_).then((items) => { return this.scopeItemsToBaseDir_(items); }); - // let fullPath = this.fullPath_(path); - // return this.driver_.list(fullPath).then((items) => { - // return items; - // // items = this.scopeItemsToBaseDir_(items); - // // if (recursive) { - // // let chain = []; - // // for (let i = 0; i < items.length; i++) { - // // let item = items[i]; - // // if (!item.isDir) continue; - - // // chain.push(() => { - // // return this.list(item.path, true).then((children) => { - // // for (let j = 0; j < children.length; j++) { - // // let md = children[j]; - // // md.path = item.path + '/' + md.path; - // // items.push(md); - // // } - // // }); - // // }); - // // } - - // // return promiseChain(chain).then(() => { - // // return items; - // // }); - // // } else { - // // return items; - // // } - // }); } setTimestamp(path, timestamp) { @@ -81,7 +53,6 @@ class FileApi { } stat(path) { - //console.info('stat ' + path); return this.driver_.stat(this.fullPath_(path)).then((output) => { if (!output) return output; output.path = path; @@ -90,12 +61,10 @@ class FileApi { } get(path) { - //console.info('get ' + path); return this.driver_.get(this.fullPath_(path)); } put(path, content) { - //console.info('put ' + path); return this.driver_.put(this.fullPath_(path), content); } diff --git a/ReactNativeClient/src/models/base-item.js b/ReactNativeClient/src/models/base-item.js index 301508ccc..3e3363754 100644 --- a/ReactNativeClient/src/models/base-item.js +++ b/ReactNativeClient/src/models/base-item.js @@ -3,6 +3,7 @@ import { Note } from 'src/models/note.js'; import { Folder } from 'src/models/folder.js'; import { folderItemFilename } from 'src/string-utils.js' import { Database } from 'src/database.js'; +import { time } from 'src/time-utils.js'; import moment from 'moment'; class BaseItem extends BaseModel { @@ -17,8 +18,15 @@ class BaseItem extends BaseModel { static itemClass(item) { if (!item) throw new Error('Item cannot be null'); - if (!('type_' in item)) throw new Error('Item does not have a type_ property'); - return item.type_ == BaseModel.ITEM_TYPE_NOTE ? Note : Folder; + + if (typeof item === 'object') { + if (!('type_' in item)) throw new Error('Item does not have a type_ property'); + return item.type_ == BaseModel.ITEM_TYPE_NOTE ? Note : Folder; + } else { + if (Number(item) === BaseModel.ITEM_TYPE_NOTE) return Note; + if (Number(item) === BaseModel.ITEM_TYPE_FOLDER) return Folder; + throw new Error('Unknown type: ' + item); + } } static pathToId(path) { @@ -34,10 +42,10 @@ class BaseItem extends BaseModel { }); } - static toFriendlyString_format(propName, propValue) { + static serialize_format(propName, propValue) { if (['created_time', 'updated_time'].indexOf(propName) >= 0) { if (!propValue) return ''; - propValue = moment.unix(propValue).utc().format('YYYY-MM-DD HH:mm:ss') + 'Z'; + propValue = moment.unix(propValue / 1000).utc().format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z'; } else if (propValue === null || propValue === undefined) { propValue = ''; } @@ -45,20 +53,22 @@ class BaseItem extends BaseModel { return propValue; } - static fromFriendlyString_format(propName, propValue) { + static unserialize_format(type, propName, propValue) { if (propName == 'type_') return propValue; + let ItemClass = this.itemClass(type); + if (['created_time', 'updated_time'].indexOf(propName) >= 0) { if (!propValue) return 0; - propValue = moment(propValue, 'YYYY-MM-DD HH:mm:ssZ').unix(); + propValue = moment(propValue, 'YYYY-MM-DDTHH:mm:ss.SSSZ').format('x'); } else { - propValue = Database.formatValue(this.fieldType(propName), propValue); + propValue = Database.formatValue(ItemClass.fieldType(propName), propValue); } return propValue; } - static toFriendlyString(item, type = null, shownKeys = null) { + static serialize(item, type = null, shownKeys = null) { let output = []; output.push(item.title); @@ -67,14 +77,14 @@ class BaseItem extends BaseModel { output.push(''); for (let i = 0; i < shownKeys.length; i++) { let v = item[shownKeys[i]]; - v = this.toFriendlyString_format(shownKeys[i], v); + v = this.serialize_format(shownKeys[i], v); output.push(shownKeys[i] + ': ' + v); } return output.join("\n"); } - static fromFriendlyString(content) { + static unserialize(content) { let lines = content.split("\n"); let output = {}; let state = 'readingProps'; @@ -94,7 +104,7 @@ class BaseItem extends BaseModel { if (p < 0) throw new Error('Invalid property format: ' + line + ": " + content); let key = line.substr(0, p).trim(); let value = line.substr(p + 1).trim(); - output[key] = this.fromFriendlyString_format(key, value); + output[key] = value; } else if (state == 'readingBody') { body.splice(0, 0, line); } @@ -104,11 +114,29 @@ class BaseItem extends BaseModel { let title = body.splice(0, 2); output.title = title[0]; + + if (!output.type_) throw new Error('Missing required property: type_: ' + content); + output.type_ = Number(output.type_); + if (output.type_ == BaseModel.ITEM_TYPE_NOTE) output.body = body.join("\n"); + for (let n in output) { + if (!output.hasOwnProperty(n)) continue; + output[n] = this.unserialize_format(output.type_, n, output[n]); + } + return output; } + static itemsThatNeedSync(limit = 100) { + return Folder.modelSelectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit).then((items) => { + if (items.length) return { hasMore: true, items: items }; + return Note.modelSelectAll('SELECT * FROM notes WHERE sync_time < updated_time LIMIT ' + limit).then((items) => { + return { hasMore: items.length >= limit, items: items }; + }); + }); + } + } export { BaseItem }; \ No newline at end of file diff --git a/ReactNativeClient/src/models/folder.js b/ReactNativeClient/src/models/folder.js index 367cc4161..0e1f24311 100644 --- a/ReactNativeClient/src/models/folder.js +++ b/ReactNativeClient/src/models/folder.js @@ -13,8 +13,8 @@ class Folder extends BaseItem { return 'folders'; } - static toFriendlyString(folder) { - return super.toFriendlyString(folder, 'folder', ['id', 'created_time', 'updated_time', 'type_']); + static serialize(folder) { + return super.serialize(folder, 'folder', ['id', 'created_time', 'updated_time', 'type_']); } static itemType() { diff --git a/ReactNativeClient/src/models/note.js b/ReactNativeClient/src/models/note.js index e5ccabdf4..cf5b1eb57 100644 --- a/ReactNativeClient/src/models/note.js +++ b/ReactNativeClient/src/models/note.js @@ -12,8 +12,8 @@ class Note extends BaseItem { return 'notes'; } - static toFriendlyString(note, type = null, shownKeys = null) { - return super.toFriendlyString(note, 'note', ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time', 'id', 'parent_id', 'type_']); + static serialize(note, type = null, shownKeys = null) { + return super.serialize(note, 'note', ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time', 'id', 'parent_id', 'type_']); } static itemType() { diff --git a/ReactNativeClient/src/synchronizer.js b/ReactNativeClient/src/synchronizer.js index 2611b1495..6b17df391 100644 --- a/ReactNativeClient/src/synchronizer.js +++ b/ReactNativeClient/src/synchronizer.js @@ -1,34 +1,17 @@ require('babel-plugin-transform-runtime'); -import { Log } from 'src/log.js'; -import { Setting } from 'src/models/setting.js'; -import { Change } from 'src/models/change.js'; -import { Folder } from 'src/models/folder.js'; -import { Note } from 'src/models/note.js'; import { BaseItem } from 'src/models/base-item.js'; -import { BaseModel } from 'src/base-model.js'; -import { promiseChain } from 'src/promise-utils.js'; -import { NoteFolderService } from 'src/services/note-folder-service.js'; -import { time } from 'src/time-utils.js'; import { sprintf } from 'sprintf-js'; -//import { promiseWhile } from 'src/promise-utils.js'; -import moment from 'moment'; - -const fs = require('fs'); -const path = require('path'); +import { time } from 'src/time-utils.js'; +import { Log } from 'src/log.js' class Synchronizer { constructor(db, api) { - this.state_ = 'idle'; this.db_ = db; this.api_ = api; } - state() { - return this.state_; - } - db() { return this.db_; } @@ -37,382 +20,101 @@ class Synchronizer { return this.api_; } - loadParentAndItem(change) { - if (change.item_type == BaseModel.ITEM_TYPE_NOTE) { - return Note.load(change.item_id).then((note) => { - if (!note) return { parent:null, item: null }; + async start() { + // ------------------------------------------------------------------------ + // First, find all the items that have been changed since the + // last sync and apply the changes to remote. + // ------------------------------------------------------------------------ - return Folder.load(note.parent_id).then((folder) => { - return Promise.resolve({ parent: folder, item: note }); - }); - }); - } else { - return Folder.load(change.item_id).then((folder) => { - return Promise.resolve({ parent: null, item: folder }); - }); - } - } - - remoteFileByPath(remoteFiles, path) { - for (let i = 0; i < remoteFiles.length; i++) { - if (remoteFiles[i].path == path) return remoteFiles[i]; - } - return null; - } - - conflictDir(remoteFiles) { - let d = this.remoteFileByPath('Conflicts'); - if (!d) { - return this.api().mkdir('Conflicts').then(() => { - return 'Conflicts'; - }); - } else { - return Promise.resolve('Conflicts'); - } - } - - moveConflict(item) { - // No need to handle folder conflicts - if (item.type == 'folder') return Promise.resolve(); - - return this.conflictDir().then((conflictDirPath) => { - let p = path.basename(item.path).split('.'); - let pos = item.type == 'folder' ? p.length - 1 : p.length - 2; - p.splice(pos, 0, moment().format('YYYYMMDDThhmmss')); - let newPath = p.join('.'); - return this.api().move(item.path, conflictDirPath + '/' + newPath); - }); - } - - itemByPath(items, path) { - for (let i = 0; i < items.length; i++) { - if (items[i].path == path) return items[i]; - } - return null; - } - - itemIsSameDate(item, date) { - return item.updatedTime === date; - } - - itemIsStrictlyNewerThan(item, date) { - return item.updatedTime > date; - } - - itemIsStrictlyOlderThan(item, date) { - return item.updatedTime < date; - } - - dbItemToSyncItem(dbItem) { - if (!dbItem) return null; - - return { - type: dbItem.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note', - path: Folder.systemPath(dbItem), - syncTime: dbItem.sync_time, - updatedTime: dbItem.updated_time, - dbItem: dbItem, - }; - } - - remoteItemToSyncItem(remoteItem) { - if (!remoteItem) return null; - - return { - type: remoteItem.content.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note', - path: remoteItem.path, - syncTime: 0, - updatedTime: remoteItem.updatedTime, - remoteItem: remoteItem, - }; - } - - syncAction(localItem, remoteItem, deletedLocalPaths) { - let output = this.syncActions(localItem ? [localItem] : [], remoteItem ? [remoteItem] : [], deletedLocalPaths); - if (output.length > 1) throw new Error('Invalid number of actions returned'); - return output.length ? output[0] : null; - } - - // Assumption: it's not possible to, for example, have a directory one the dest - // and a file with the same name on the source. It's not possible because the - // file and directory names are UUID so should be unique. - // Each item must have these properties: - // - path - // - type - // - syncTime - // - updatedTime - syncActions(localItems, remoteItems, deletedLocalPaths) { - let output = []; let donePaths = []; - - // console.info('=================================================='); - // console.info(localItems, remoteItems); - - for (let i = 0; i < localItems.length; i++) { - let local = localItems[i]; - let remote = this.itemByPath(remoteItems, local.path); - - let action = { - local: local, - remote: remote, - }; - - if (!remote) { - if (local.syncTime) { - action.type = 'delete'; - action.dest = 'local'; - action.reason = 'Local has been synced to remote previously, but remote no longer exist, which means remote has been deleted'; - } else { - action.type = 'create'; - action.dest = 'remote'; - action.reason = 'Local has never been synced to remote, and remote does not exists, which means remote must be created'; - } - } else { - if (this.itemIsStrictlyOlderThan(local, local.syncTime)) continue; - - if (this.itemIsStrictlyOlderThan(remote, local.updatedTime)) { - action.type = 'update'; - action.dest = 'remote'; - action.reason = sprintf('Remote (%s) was modified before updated time of local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString(),); - } else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && this.itemIsStrictlyNewerThan(local, local.syncTime)) { - action.type = 'conflict'; - action.reason = sprintf('Both remote (%s) and local (%s) were modified after the last sync (%s).', - moment.unix(remote.updatedTime).toISOString(), - moment.unix(local.updatedTime).toISOString(), - moment.unix(local.syncTime).toISOString() - ); - - if (local.type == 'folder') { - action.solution = [ - { type: 'update', dest: 'local' }, - ]; - } else { - action.solution = [ - { type: 'copy-to-remote-conflict-dir', dest: 'local' }, - { type: 'copy-to-local-conflict-dir', dest: 'local' }, - { type: 'update', dest: 'local' }, - ]; - } - } else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && local.updatedTime <= local.syncTime) { - action.type = 'update'; - action.dest = 'local'; - action.reason = sprintf('Remote (%s) was modified after update time of local (%s). And sync time (%s) is the same or more recent than local update time', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString()); - } else { - continue; // Neither local nor remote item have been changed recently - } - } - - donePaths.push(local.path); - - output.push(action); - } - - for (let i = 0; i < remoteItems.length; i++) { - let remote = remoteItems[i]; - if (donePaths.indexOf(remote.path) >= 0) continue; // Already handled in the previous loop - let local = this.itemByPath(localItems, remote.path); - - let action = { - local: local, - remote: remote, - }; - - if (!local) { - if (deletedLocalPaths.indexOf(remote.path) >= 0) { - action.type = 'delete'; - action.dest = 'remote'; - } else { - action.type = 'create'; - action.dest = 'local'; - } - } else { - if (this.itemIsStrictlyOlderThan(remote, local.syncTime)) continue; // Already have this version - - // Note: no conflict is possible here since if the local item has been - // modified since the last sync, it's been processed in the previous loop. - // So throw an exception is this normally impossible condition happens anyway. - // It's handled at condition this.itemIsStrictlyNewerThan(remote, local.syncTime) in above loop - if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) { - console.error('Remote cannot be newer than last sync time', remote, local); - throw new Error('Remote cannot be newer than last sync time'); - } - - if (this.itemIsStrictlyNewerThan(remote, local.updatedTime)) { - action.type = 'update'; - action.dest = 'local'; - action.reason = sprintf('Remote (%s) was modified after local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.updatedTime).toISOString(),);; - } else { - continue; - } - } - - output.push(action); - } - - // console.info('-----------------------------------------'); - // console.info(output); - - return output; - } - - processState(state) { - Log.info('Sync: processing: ' + state); - this.state_ = state; - - if (state == 'uploadChanges') { - return this.processState_uploadChanges(); - } else if (state == 'downloadChanges') { - //return this.processState('idle'); - return this.processState_downloadChanges(); - } else if (state == 'idle') { - // Nothing - return Promise.resolve(); - } else { - throw new Error('Invalid state: ' . state); - } - } - - processSyncAction(action) { - //console.info('Sync action: ', action); - //console.info('Sync action: ' + JSON.stringify(action)); - - if (!action) return Promise.resolve(); - - console.info('Sync action: ' + action.type + ' ' + action.dest + ': ' + action.reason); - - if (action.type == 'conflict') { - console.info(action); - - } else { - let syncItem = action[action.dest == 'local' ? 'remote' : 'local']; - let path = syncItem.path; - - if (action.type == 'create') { - if (action.dest == 'remote') { - let content = null; - let dbItem = syncItem.dbItem; - - if (syncItem.type == 'folder') { - content = Folder.toFriendlyString(dbItem); - } else { - content = Note.toFriendlyString(dbItem); - } - - return this.api().put(path, content).then(() => { - return this.api().setTimestamp(path, dbItem.updated_time); - }); - - // TODO: save sync_time - } else { - let dbItem = syncItem.remoteItem.content; - dbItem.sync_time = time.unix(); - dbItem.updated_time = action.remote.updatedTime; - if (syncItem.type == 'folder') { - return Folder.save(dbItem, { isNew: true, autoTimestamp: false }); - } else { - return Note.save(dbItem, { isNew: true, autoTimestamp: false }); - } - - // TODO: save sync_time - } - } - - if (action.type == 'update') { - if (action.dest == 'remote') { - let dbItem = syncItem.dbItem; - let ItemClass = BaseItem.itemClass(dbItem); - let content = ItemClass.toFriendlyString(dbItem); - //console.info('PUT', content); - return this.api().put(path, content).then(() => { - return this.api().setTimestamp(path, dbItem.updated_time); - }).then(() => { - let toSave = { id: dbItem.id, sync_time: time.unix() }; - return NoteFolderService.save(syncItem.type, dbItem, null, { autoTimestamp: false }); - }); - } else { - let dbItem = Object.assign({}, syncItem.remoteItem.content); - dbItem.sync_time = time.unix(); - return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem, { autoTimestamp: false }); - } - } - } - - return Promise.resolve(); // TODO - } - - async processLocalItem(dbItem) { - let localItem = this.dbItemToSyncItem(dbItem); - - let remoteItem = await this.api().stat(localItem.path); - let action = this.syncAction(localItem, remoteItem, []); - await this.processSyncAction(action); - - let toSave = Object.assign({}, dbItem); - toSave.sync_time = time.unix(); - return NoteFolderService.save(localItem.type, toSave, dbItem, { autoTimestamp: false }); - } - - async processRemoteItem(remoteItem) { - let content = await this.api().get(remoteItem.path); - if (!content) throw new Error('Cannot get content for: ' + remoteItem.path); - remoteItem.content = Note.fromFriendlyString(content); - let remoteSyncItem = this.remoteItemToSyncItem(remoteItem); - - let dbItem = await BaseItem.loadItemByPath(remoteItem.path); - let localSyncItem = this.dbItemToSyncItem(dbItem); - - let action = this.syncAction(localSyncItem, remoteSyncItem, []); - return this.processSyncAction(action); - } - - async processState_uploadChanges() { while (true) { - let result = await NoteFolderService.itemsThatNeedSync(50); - console.info('Items that need sync: ' + result.items.length); - for (let i = 0; i < result.items.length; i++) { - let item = result.items[i]; - await this.processLocalItem(item); + let result = await BaseItem.itemsThatNeedSync(); + let locals = result.items; + + for (let i = 0; i < locals.length; i++) { + let local = locals[i]; + let ItemClass = BaseItem.itemClass(local); + let path = BaseItem.systemPath(local); + + // Safety check to avoid infinite loops: + if (donePaths.indexOf(path) > 0) throw new Error(sprintf('Processing a path that has already been done: %s. sync_time was not updated?', path)); + + let remote = await this.api().stat(path); + let content = ItemClass.serialize(local); + let action = null; + + if (!remote) { + action = 'createRemote'; + } else { + if (remote.updated_time > local.updated_time) { + action = 'conflict'; + } else { + action = 'updateRemote'; + } + } + + if (action == 'createRemote' || action == 'updateRemote') { + await this.api().put(path, content); + await this.api().setTimestamp(path, local.updated_time); + } else if (action == 'conflict') { + console.warn('FOUND CONFLICT', local, remote); + } + + let newLocal = { id: local.id, sync_time: time.unixMs(), type_: local.type_ }; + await ItemClass.save(newLocal, { autoTimestamp: false }); + + donePaths.push(path); } if (!result.hasMore) break; } - //console.info('DOWNLOAD CHANGE DISABLED'); return Promise.resolve(); - - return this.processState('downloadChanges'); - } + // ------------------------------------------------------------------------ + // Then, loop through all the remote items, find those that + // have been updated, and apply the changes to local. + // ------------------------------------------------------------------------ - async processState_downloadChanges() { - let items = await this.api().list(); - for (let i = 0; i < items.length; i++) { - await this.processRemoteItem(items[i]); + // At this point all the local items that have changed have been pushed to remote + // or handled as conflicts, so no conflict is possible after this. + + let remotes = await this.api().list(); + for (let i = 0; i < remotes.length; i++) { + let remote = remotes[i]; + let path = remote.path; + if (donePaths.indexOf(path) > 0) continue; + + let action = null; + let local = await BaseItem.loadItemByPath(path); + if (!local) { + action = 'createLocal'; + } else { + if (remote.updated_time > local.updated_time) { + action = 'updateLocal'; + } + } + + if (!action) continue; + + if (action == 'createLocal' || action == 'updateLocal') { + let content = await this.api().get(path); + if (!content) { + Log.warn('Remote item has been deleted between now and the list() call? In that case it will handled during the next sync: ' + path); + continue; + } + content = BaseItem.unserialize(content); + let ItemClass = BaseItem.itemClass(content); + + content.sync_time = time.unixMs(); + let options = { autoTimestamp: false }; + if (action == 'createLocal') options.isNew = true; + await ItemClass.save(content, options); + } } - return this.processState('idle'); + + return Promise.resolve(); } - start() { - Log.info('Sync: start'); - - if (this.state() != 'idle') { - return Promise.reject('Cannot start synchronizer because synchronization already in progress. State: ' + this.state()); - } - - this.state_ = 'started'; - - // if (!this.api().session()) { - // Log.info("Sync: cannot start synchronizer because user is not logged in."); - // return; - // } - - return this.processState('uploadChanges').catch((error) => { - console.info('Synchronizer error:', error); - throw error; - }); - } - - - } export { Synchronizer }; \ No newline at end of file diff --git a/ReactNativeClient/src/synchronizer_old.js b/ReactNativeClient/src/synchronizer_old.js index 507d5be6b..aa583599e 100644 --- a/ReactNativeClient/src/synchronizer_old.js +++ b/ReactNativeClient/src/synchronizer_old.js @@ -65,7 +65,7 @@ class Synchronizer { } else if (c.type == Change.TYPE_CREATE) { p = this.loadParentAndItem(c).then((result) => { let options = { - contents: Note.toFriendlyString(result.item), + contents: Note.serialize(result.item), path: Note.systemPath(result.parent, result.item), mode: 'overwrite', // client_modified: @@ -79,7 +79,7 @@ class Synchronizer { // console.info(item); // let options = { - // contents: Note.toFriendlyString(item), + // contents: Note.serialize(item), // path: Note.systemPath(item), // mode: 'overwrite', // // client_modified: @@ -87,7 +87,7 @@ class Synchronizer { // // console.info(options); - // //let content = Note.toFriendlyString(item); + // //let content = Note.serialize(item); // //console.info(content); // //console.info('SYNC', item); diff --git a/ReactNativeClient/src/synchronizer_old2.js b/ReactNativeClient/src/synchronizer_old2.js new file mode 100644 index 000000000..950e5c51b --- /dev/null +++ b/ReactNativeClient/src/synchronizer_old2.js @@ -0,0 +1,418 @@ +require('babel-plugin-transform-runtime'); + +import { Log } from 'src/log.js'; +import { Setting } from 'src/models/setting.js'; +import { Change } from 'src/models/change.js'; +import { Folder } from 'src/models/folder.js'; +import { Note } from 'src/models/note.js'; +import { BaseItem } from 'src/models/base-item.js'; +import { BaseModel } from 'src/base-model.js'; +import { promiseChain } from 'src/promise-utils.js'; +import { NoteFolderService } from 'src/services/note-folder-service.js'; +import { time } from 'src/time-utils.js'; +import { sprintf } from 'sprintf-js'; +//import { promiseWhile } from 'src/promise-utils.js'; +import moment from 'moment'; + +const fs = require('fs'); +const path = require('path'); + +class Synchronizer { + + constructor(db, api) { + this.state_ = 'idle'; + this.db_ = db; + this.api_ = api; + } + + state() { + return this.state_; + } + + db() { + return this.db_; + } + + api() { + return this.api_; + } + + loadParentAndItem(change) { + if (change.item_type == BaseModel.ITEM_TYPE_NOTE) { + return Note.load(change.item_id).then((note) => { + if (!note) return { parent:null, item: null }; + + return Folder.load(note.parent_id).then((folder) => { + return Promise.resolve({ parent: folder, item: note }); + }); + }); + } else { + return Folder.load(change.item_id).then((folder) => { + return Promise.resolve({ parent: null, item: folder }); + }); + } + } + + remoteFileByPath(remoteFiles, path) { + for (let i = 0; i < remoteFiles.length; i++) { + if (remoteFiles[i].path == path) return remoteFiles[i]; + } + return null; + } + + conflictDir(remoteFiles) { + let d = this.remoteFileByPath('Conflicts'); + if (!d) { + return this.api().mkdir('Conflicts').then(() => { + return 'Conflicts'; + }); + } else { + return Promise.resolve('Conflicts'); + } + } + + moveConflict(item) { + // No need to handle folder conflicts + if (item.type == 'folder') return Promise.resolve(); + + return this.conflictDir().then((conflictDirPath) => { + let p = path.basename(item.path).split('.'); + let pos = item.type == 'folder' ? p.length - 1 : p.length - 2; + p.splice(pos, 0, moment().format('YYYYMMDDThhmmss')); + let newPath = p.join('.'); + return this.api().move(item.path, conflictDirPath + '/' + newPath); + }); + } + + itemByPath(items, path) { + for (let i = 0; i < items.length; i++) { + if (items[i].path == path) return items[i]; + } + return null; + } + + itemIsSameDate(item, date) { + return item.updated_time === date; + } + + itemIsStrictlyNewerThan(item, date) { + return item.updated_time > date; + } + + itemIsStrictlyOlderThan(item, date) { + return item.updated_time < date; + } + + dbItemToSyncItem(dbItem) { + if (!dbItem) return null; + + return { + type: dbItem.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note', + path: Folder.systemPath(dbItem), + syncTime: dbItem.sync_time, + updated_time: dbItem.updated_time, + dbItem: dbItem, + }; + } + + remoteItemToSyncItem(remoteItem) { + if (!remoteItem) return null; + + return { + type: remoteItem.content.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note', + path: remoteItem.path, + syncTime: 0, + updated_time: remoteItem.updated_time, + remoteItem: remoteItem, + }; + } + + syncAction(localItem, remoteItem, deletedLocalPaths) { + let output = this.syncActions(localItem ? [localItem] : [], remoteItem ? [remoteItem] : [], deletedLocalPaths); + if (output.length > 1) throw new Error('Invalid number of actions returned'); + return output.length ? output[0] : null; + } + + // Assumption: it's not possible to, for example, have a directory one the dest + // and a file with the same name on the source. It's not possible because the + // file and directory names are UUID so should be unique. + // Each item must have these properties: + // - path + // - type + // - syncTime + // - updated_time + syncActions(localItems, remoteItems, deletedLocalPaths) { + let output = []; + let donePaths = []; + + // console.info('=================================================='); + // console.info(localItems, remoteItems); + + for (let i = 0; i < localItems.length; i++) { + let local = localItems[i]; + let remote = this.itemByPath(remoteItems, local.path); + + let action = { + local: local, + remote: remote, + }; + + if (!remote) { + if (local.syncTime) { + action.type = 'delete'; + action.dest = 'local'; + action.reason = 'Local has been synced to remote previously, but remote no longer exist, which means remote has been deleted'; + } else { + action.type = 'create'; + action.dest = 'remote'; + action.reason = 'Local has never been synced to remote, and remote does not exists, which means remote must be created'; + } + } else { + if (this.itemIsStrictlyOlderThan(local, local.syncTime)) continue; + + if (this.itemIsStrictlyOlderThan(remote, local.updated_time)) { + action.type = 'update'; + action.dest = 'remote'; + action.reason = sprintf('Remote (%s) was modified before updated time of local (%s).', moment.unix(remote.updated_time).toISOString(), moment.unix(local.syncTime).toISOString(),); + } else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && this.itemIsStrictlyNewerThan(local, local.syncTime)) { + action.type = 'conflict'; + action.reason = sprintf('Both remote (%s) and local (%s) were modified after the last sync (%s).', + moment.unix(remote.updated_time).toISOString(), + moment.unix(local.updated_time).toISOString(), + moment.unix(local.syncTime).toISOString() + ); + + if (local.type == 'folder') { + action.solution = [ + { type: 'update', dest: 'local' }, + ]; + } else { + action.solution = [ + { type: 'copy-to-remote-conflict-dir', dest: 'local' }, + { type: 'copy-to-local-conflict-dir', dest: 'local' }, + { type: 'update', dest: 'local' }, + ]; + } + } else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && local.updated_time <= local.syncTime) { + action.type = 'update'; + action.dest = 'local'; + action.reason = sprintf('Remote (%s) was modified after update time of local (%s). And sync time (%s) is the same or more recent than local update time', moment.unix(remote.updated_time).toISOString(), moment.unix(local.updated_time).toISOString(), moment.unix(local.syncTime).toISOString()); + } else { + continue; // Neither local nor remote item have been changed recently + } + } + + donePaths.push(local.path); + + output.push(action); + } + + for (let i = 0; i < remoteItems.length; i++) { + let remote = remoteItems[i]; + if (donePaths.indexOf(remote.path) >= 0) continue; // Already handled in the previous loop + let local = this.itemByPath(localItems, remote.path); + + let action = { + local: local, + remote: remote, + }; + + if (!local) { + if (deletedLocalPaths.indexOf(remote.path) >= 0) { + action.type = 'delete'; + action.dest = 'remote'; + } else { + action.type = 'create'; + action.dest = 'local'; + } + } else { + if (this.itemIsStrictlyOlderThan(remote, local.syncTime)) continue; // Already have this version + + // Note: no conflict is possible here since if the local item has been + // modified since the last sync, it's been processed in the previous loop. + // So throw an exception is this normally impossible condition happens anyway. + // It's handled at condition this.itemIsStrictlyNewerThan(remote, local.syncTime) in above loop + if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) { + console.error('Remote cannot be newer than last sync time', remote, local); + throw new Error('Remote cannot be newer than last sync time'); + } + + if (this.itemIsStrictlyNewerThan(remote, local.updated_time)) { + action.type = 'update'; + action.dest = 'local'; + action.reason = sprintf('Remote (%s) was modified after local (%s).', moment.unix(remote.updated_time).toISOString(), moment.unix(local.updated_time).toISOString(),);; + } else { + continue; + } + } + + output.push(action); + } + + // console.info('-----------------------------------------'); + // console.info(output); + + return output; + } + + processState(state) { + Log.info('Sync: processing: ' + state); + this.state_ = state; + + if (state == 'uploadChanges') { + return this.processState_uploadChanges(); + } else if (state == 'downloadChanges') { + //return this.processState('idle'); + return this.processState_downloadChanges(); + } else if (state == 'idle') { + // Nothing + return Promise.resolve(); + } else { + throw new Error('Invalid state: ' . state); + } + } + + processSyncAction(action) { + //console.info('Sync action: ', action); + //console.info('Sync action: ' + JSON.stringify(action)); + + if (!action) return Promise.resolve(); + + console.info('Sync action: ' + action.type + ' ' + action.dest + ': ' + action.reason); + + if (action.type == 'conflict') { + console.info(action); + + } else { + let syncItem = action[action.dest == 'local' ? 'remote' : 'local']; + let path = syncItem.path; + + if (action.type == 'create') { + if (action.dest == 'remote') { + let content = null; + let dbItem = syncItem.dbItem; + + if (syncItem.type == 'folder') { + content = Folder.serialize(dbItem); + } else { + content = Note.serialize(dbItem); + } + + return this.api().put(path, content).then(() => { + return this.api().setTimestamp(path, dbItem.updated_time); + }); + + // TODO: save sync_time + } else { + let dbItem = syncItem.remoteItem.content; + dbItem.sync_time = time.unix(); + dbItem.updated_time = action.remote.updated_time; + if (syncItem.type == 'folder') { + return Folder.save(dbItem, { isNew: true, autoTimestamp: false }); + } else { + return Note.save(dbItem, { isNew: true, autoTimestamp: false }); + } + + // TODO: save sync_time + } + } + + if (action.type == 'update') { + if (action.dest == 'remote') { + let dbItem = syncItem.dbItem; + let ItemClass = BaseItem.itemClass(dbItem); + let content = ItemClass.serialize(dbItem); + //console.info('PUT', content); + return this.api().put(path, content).then(() => { + return this.api().setTimestamp(path, dbItem.updated_time); + }).then(() => { + let toSave = { id: dbItem.id, sync_time: time.unix() }; + return NoteFolderService.save(syncItem.type, dbItem, null, { autoTimestamp: false }); + }); + } else { + let dbItem = Object.assign({}, syncItem.remoteItem.content); + dbItem.sync_time = time.unix(); + return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem, { autoTimestamp: false }); + } + } + } + + return Promise.resolve(); // TODO + } + + async processLocalItem(dbItem) { + let localItem = this.dbItemToSyncItem(dbItem); + + let remoteItem = await this.api().stat(localItem.path); + let action = this.syncAction(localItem, remoteItem, []); + await this.processSyncAction(action); + + let toSave = Object.assign({}, dbItem); + toSave.sync_time = time.unix(); + return NoteFolderService.save(localItem.type, toSave, dbItem, { autoTimestamp: false }); + } + + async processRemoteItem(remoteItem) { + let content = await this.api().get(remoteItem.path); + if (!content) throw new Error('Cannot get content for: ' + remoteItem.path); + remoteItem.content = Note.unserialize(content); + let remoteSyncItem = this.remoteItemToSyncItem(remoteItem); + + let dbItem = await BaseItem.loadItemByPath(remoteItem.path); + let localSyncItem = this.dbItemToSyncItem(dbItem); + + let action = this.syncAction(localSyncItem, remoteSyncItem, []); + return this.processSyncAction(action); + } + + async processState_uploadChanges() { + while (true) { + let result = await NoteFolderService.itemsThatNeedSync(50); + console.info('Items that need sync: ' + result.items.length); + for (let i = 0; i < result.items.length; i++) { + let item = result.items[i]; + await this.processLocalItem(item); + } + + if (!result.hasMore) break; + } + + //console.info('DOWNLOAD CHANGE DISABLED'); return Promise.resolve(); + + return this.processState('downloadChanges'); + } + + async processState_downloadChanges() { + let items = await this.api().list(); + for (let i = 0; i < items.length; i++) { + await this.processRemoteItem(items[i]); + } + + return this.processState('idle'); + } + + start() { + Log.info('Sync: start'); + + if (this.state() != 'idle') { + return Promise.reject('Cannot start synchronizer because synchronization already in progress. State: ' + this.state()); + } + + this.state_ = 'started'; + + // if (!this.api().session()) { + // Log.info("Sync: cannot start synchronizer because user is not logged in."); + // return; + // } + + return this.processState('uploadChanges').catch((error) => { + console.info('Synchronizer error:', error); + throw error; + }); + } + + + +} + +export { Synchronizer }; \ No newline at end of file diff --git a/ReactNativeClient/src/time-utils.js b/ReactNativeClient/src/time-utils.js index 8557670c0..6d588df9a 100644 --- a/ReactNativeClient/src/time-utils.js +++ b/ReactNativeClient/src/time-utils.js @@ -2,6 +2,14 @@ let time = { unix() { return Math.round((new Date()).getTime() / 1000); + }, + + unixMs() { + return (new Date()).getTime(); + }, + + unixMsToS(ms) { + return Math.round(ms / 1000); } }