self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-10-31 00:07:48 +02:00
Code Issues Releases Activity

refactored sync class

Browse Source
This commit is contained in:
Laurent Cozic
2017-06-18 23:06:10 +01:00
parent f9480cb882
commit 02ff02a9d9
19 changed files with 628 additions and 690 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CliClient/app/cmd.js
View File

@@ -463,9 +463,9 @@ async function createLocalItems() {
// } // }
// if (!currentFolder) { // if (!currentFolder) {
// this.log(Folder.toFriendlyString(item)); // this.log(Folder.serialize(item));
// } else { // } else {
// this.log(Note.toFriendlyString(item)); // this.log(Note.serialize(item));
// } // }
// }).catch((error) => { // }).catch((error) => {
// this.log(error); // this.log(error);

10
CliClient/app/import-enex.js
View File

@@ -527,7 +527,7 @@ function saveNoteToWebApi(note) {
}); });
} }
function noteToFriendlyString_format(propName, propValue) { function noteserialize_format(propName, propValue) {
if (['created_time', 'updated_time'].indexOf(propName) >= 0) { if (['created_time', 'updated_time'].indexOf(propName) >= 0) {
if (!propValue) return ''; if (!propValue) return '';
propValue = moment.unix(propValue).format('YYYY-MM-DD hh:mm:ss'); propValue = moment.unix(propValue).format('YYYY-MM-DD hh:mm:ss');
@@ -538,7 +538,7 @@ function noteToFriendlyString_format(propName, propValue) {
return propValue; return propValue;
} }
function noteToFriendlyString(note) { function noteserialize(note) {
let shownKeys = ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time']; let shownKeys = ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time'];
let output = []; let output = [];
@@ -548,7 +548,7 @@ function noteToFriendlyString(note) {
output.push(''); output.push('');
for (let i = 0; i < shownKeys.length; i++) { for (let i = 0; i < shownKeys.length; i++) {
let v = note[shownKeys[i]]; let v = note[shownKeys[i]];
v = noteToFriendlyString_format(shownKeys[i], v); v = noteserialize_format(shownKeys[i], v);
output.push(shownKeys[i] + ': ' + v); output.push(shownKeys[i] + ': ' + v);
} }
@@ -623,7 +623,7 @@ const baseNoteDir = '/home/laurent/Temp/TestImport';
// }); // });
function saveNoteToDisk(folder, note) { function saveNoteToDisk(folder, note) {
const noteContent = noteToFriendlyString(note); const noteContent = noteserialize(note);
const notePath = baseNoteDir + '/' + folderFilename(folder) + '/' + noteFilename(note); const notePath = baseNoteDir + '/' + folderFilename(folder) + '/' + noteFilename(note);
// console.info('==================================================='); // console.info('===================================================');
@@ -694,7 +694,7 @@ function importEnex(parentFolder, stream) {
saveNoteToDisk(parentFolder, note); saveNoteToDisk(parentFolder, note);
// console.info(noteToFriendlyString(note)); // console.info(noteserialize(note));
// console.info('========================================================================================================================='); // console.info('=========================================================================================================================');
//saveNoteToWebApi(note); //saveNoteToWebApi(note);

277
CliClient/tests/synchronizer.js
View File

@@ -6,8 +6,16 @@ import { Note } from 'src/models/note.js';
import { BaseItem } from 'src/models/base-item.js'; import { BaseItem } from 'src/models/base-item.js';
import { BaseModel } from 'src/base-model.js'; import { BaseModel } from 'src/base-model.js';
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
// application specific logging, throwing an error, or other logic here
});
async function localItemsSameAsRemote(locals, expect) { async function localItemsSameAsRemote(locals, expect) {
try { try {
let files = await fileApi().list();
expect(locals.length).toBe(files.length);
for (let i = 0; i < locals.length; i++) { for (let i = 0; i < locals.length; i++) {
let dbItem = locals[i]; let dbItem = locals[i];
let path = BaseItem.systemPath(dbItem); let path = BaseItem.systemPath(dbItem);
@@ -19,10 +27,10 @@ async function localItemsSameAsRemote(locals, expect) {
// console.info('======================='); // console.info('=======================');
expect(!!remote).toBe(true); expect(!!remote).toBe(true);
expect(remote.updatedTime).toBe(dbItem.updated_time); expect(remote.updated_time).toBe(dbItem.updated_time);
let remoteContent = await fileApi().get(path); let remoteContent = await fileApi().get(path);
remoteContent = dbItem.type_ == BaseModel.ITEM_TYPE_NOTE ? Note.fromFriendlyString(remoteContent) : Folder.fromFriendlyString(remoteContent); remoteContent = dbItem.type_ == BaseModel.ITEM_TYPE_NOTE ? Note.unserialize(remoteContent) : Folder.unserialize(remoteContent);
expect(remoteContent.title).toBe(dbItem.title); expect(remoteContent.title).toBe(dbItem.title);
} }
} catch (error) { } catch (error) {
@@ -39,94 +47,72 @@ describe('Synchronizer', function() {
done(); done();
}); });
// it('should create remote items', async (done) => { it('should create remote items', async (done) => {
// let folder = await Folder.save({ title: "folder1" }); let folder = await Folder.save({ title: "folder1" });
// await Note.save({ title: "un", parent_id: folder.id }); await Note.save({ title: "un", parent_id: folder.id });
// let all = await Folder.all(true); let all = await Folder.all(true);
// await synchronizer().start(); await synchronizer().start();
// await localItemsSameAsRemote(all, expect); await localItemsSameAsRemote(all, expect);
// done(); done();
// }); });
// it('should update remote item', async (done) => { it('should update remote item', async (done) => {
// let folder = await Folder.save({ title: "folder1" }); let folder = await Folder.save({ title: "folder1" });
// let note = await Note.save({ title: "un", parent_id: folder.id }); let note = await Note.save({ title: "un", parent_id: folder.id });
await synchronizer().start();
// await sleep(1); await sleep(1);
// await Note.save({ title: "un UPDATE", id: note.id }); await Note.save({ title: "un UPDATE", id: note.id });
// let all = await Folder.all(true); let all = await Folder.all(true);
// await synchronizer().start(); await synchronizer().start();
// await localItemsSameAsRemote(all, expect); await localItemsSameAsRemote(all, expect);
// done(); done();
// }); });
// it('should create local items', async (done) => { it('should create local items', async (done) => {
// let folder = await Folder.save({ title: "folder1" }); let folder = await Folder.save({ title: "folder1" });
// await Note.save({ title: "un", parent_id: folder.id }); await Note.save({ title: "un", parent_id: folder.id });
// await synchronizer().start(); await synchronizer().start();
// await clearDatabase();
// await synchronizer().start();
// let all = await Folder.all(true); switchClient(2);
// await localItemsSameAsRemote(all, expect);
// done(); await synchronizer().start();
// });
// it('should create same items on client 2', async (done) => { let all = await Folder.all(true);
// let folder = await Folder.save({ title: "folder1" }); await localItemsSameAsRemote(all, expect);
// let note = await Note.save({ title: "un", parent_id: folder.id });
// await synchronizer().start();
// await sleep(1); done();
});
// switchClient(2);
// await synchronizer().start();
// let folder2 = await Folder.load(folder.id);
// let note2 = await Note.load(note.id);
// expect(!!folder2).toBe(true);
// expect(!!note2).toBe(true);
// expect(folder.title).toBe(folder.title);
// expect(folder.updated_time).toBe(folder.updated_time);
// expect(note.title).toBe(note.title);
// expect(note.updated_time).toBe(note.updated_time);
// expect(note.body).toBe(note.body);
// done();
// });
it('should update local items', async (done) => { it('should update local items', async (done) => {
let folder1 = await Folder.save({ title: "folder1" }); let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: "un", parent_id: folder1.id }); let note1 = await Note.save({ title: "un", parent_id: folder1.id });
await synchronizer().start(); await synchronizer().start();
await sleep(1);
switchClient(2); switchClient(2);
await synchronizer().start(); await synchronizer().start();
await sleep(1);
let note2 = await Note.load(note1.id); let note2 = await Note.load(note1.id);
note2.title = "Updated on client 2"; note2.title = "Updated on client 2";
await Note.save(note2); await Note.save(note2);
let all = await Folder.all(true); note2 = await Note.load(note2.id);
await synchronizer().start(); await synchronizer().start();
let files = await fileApi().list();
switchClient(1); switchClient(1);
await synchronizer().start(); await synchronizer().start();
@@ -140,177 +126,4 @@ describe('Synchronizer', function() {
done(); done();
}); });
}); });
// // Note: set 1 matches set 1 of createRemoteItems()
// function createLocalItems(id, updatedTime, lastSyncTime) {
// let output = [];
// if (id === 1) {
// output.push({ path: 'test', isDir: true, updatedTime: updatedTime, lastSyncTime: lastSyncTime });
// output.push({ path: 'test/un', updatedTime: updatedTime, lastSyncTime: lastSyncTime });
// } else {
// throw new Error('Invalid ID');
// }
// return output;
// }
// function createRemoteItems(id = 1, updatedTime = null) {
// if (!updatedTime) updatedTime = time.unix();
// if (id === 1) {
// return fileApi().format()
// .then(() => fileApi().mkdir('test'))
// .then(() => fileApi().put('test/un', 'abcd'))
// .then(() => fileApi().list('', true))
// .then((items) => {
// for (let i = 0; i < items.length; i++) {
// items[i].updatedTime = updatedTime;
// }
// return items;
// });
// } else {
// throw new Error('Invalid ID');
// }
// }
// describe('Synchronizer syncActions', function() {
// beforeEach(function(done) {
// setupDatabaseAndSynchronizer(done);
// });
// it('should create remote items', function() {
// let localItems = createLocalItems(1, time.unix(), 0);
// let remoteItems = [];
// let actions = synchronizer().syncActions(localItems, remoteItems, []);
// expect(actions.length).toBe(2);
// for (let i = 0; i < actions.length; i++) {
// expect(actions[i].type).toBe('create');
// expect(actions[i].dest).toBe('remote');
// }
// });
// it('should update remote items', function(done) {
// createRemoteItems(1).then((remoteItems) => {
// let lastSyncTime = time.unix() + 1000;
// let localItems = createLocalItems(1, lastSyncTime + 1000, lastSyncTime);
// let actions = synchronizer().syncActions(localItems, remoteItems, []);
// expect(actions.length).toBe(2);
// for (let i = 0; i < actions.length; i++) {
// expect(actions[i].type).toBe('update');
// expect(actions[i].dest).toBe('remote');
// }
// done();
// });
// });
// it('should detect conflict', function(done) {
// // Simulate this scenario:
// // - Client 1 create items
// // - Client 1 sync
// // - Client 2 sync
// // - Client 2 change items
// // - Client 2 sync
// // - Client 1 change items
// // - Client 1 sync
// // => Conflict
// createRemoteItems(1).then((remoteItems) => {
// let localItems = createLocalItems(1, time.unix() + 1000, time.unix() - 1000);
// let actions = synchronizer().syncActions(localItems, remoteItems, []);
// expect(actions.length).toBe(2);
// for (let i = 0; i < actions.length; i++) {
// expect(actions[i].type).toBe('conflict');
// }
// done();
// });
// });
// it('should create local file', function(done) {
// createRemoteItems(1).then((remoteItems) => {
// let localItems = [];
// let actions = synchronizer().syncActions(localItems, remoteItems, []);
// expect(actions.length).toBe(2);
// for (let i = 0; i < actions.length; i++) {
// expect(actions[i].type).toBe('create');
// expect(actions[i].dest).toBe('local');
// }
// done();
// });
// });
// it('should delete remote files', function(done) {
// createRemoteItems(1).then((remoteItems) => {
// let localItems = createLocalItems(1, time.unix(), time.unix());
// let deletedItemPaths = [localItems[0].path, localItems[1].path];
// let actions = synchronizer().syncActions([], remoteItems, deletedItemPaths);
// expect(actions.length).toBe(2);
// for (let i = 0; i < actions.length; i++) {
// expect(actions[i].type).toBe('delete');
// expect(actions[i].dest).toBe('remote');
// }
// done();
// });
// });
// it('should delete local files', function(done) {
// let lastSyncTime = time.unix();
// createRemoteItems(1, lastSyncTime - 1000).then((remoteItems) => {
// let localItems = createLocalItems(1, lastSyncTime - 1000, lastSyncTime);
// let actions = synchronizer().syncActions(localItems, [], []);
// expect(actions.length).toBe(2);
// for (let i = 0; i < actions.length; i++) {
// expect(actions[i].type).toBe('delete');
// expect(actions[i].dest).toBe('local');
// }
// done();
// });
// });
// it('should update local files', function(done) {
// let lastSyncTime = time.unix();
// createRemoteItems(1, lastSyncTime + 1000).then((remoteItems) => {
// let localItems = createLocalItems(1, lastSyncTime - 1000, lastSyncTime);
// let actions = synchronizer().syncActions(localItems, remoteItems, []);
// expect(actions.length).toBe(2);
// for (let i = 0; i < actions.length; i++) {
// expect(actions[i].type).toBe('update');
// expect(actions[i].dest).toBe('local');
// }
// done();
// });
// });
// });
// // describe('Synchronizer start', function() {
// // beforeEach(function(done) {
// // setupDatabaseAndSynchronizer(done);
// // });
// // it('should create remote items', function(done) {
// // createFoldersAndNotes().then(() => {
// // return synchronizer().start();
// // }
// // }).then(() => {
// // done();
// // });
// // });

10
QtClient/JoplinQtClient/cliapplication.cpp
View File

@@ -135,13 +135,13 @@ void CliApplication::saveNoteIfFileChanged(Note& note, const QDateTime& original
// if (propKey.isEmpty()) { // if (propKey.isEmpty()) {
// QStringList propKeys = settings.allKeys(); // QStringList propKeys = settings.allKeys();
// for (int i = 0; i < propKeys.size(); i++) { // for (int i = 0; i < propKeys.size(); i++) {
// qStdout() << settings.keyValueToFriendlyString(propKeys[i]) << endl; // qStdout() << settings.keyValueserialize(propKeys[i]) << endl;
// } // }
// return 0; // return 0;
// } // }
// if (propValue.isEmpty()) { // if (propValue.isEmpty()) {
// qStdout() << settings.keyValueToFriendlyString(propKey) << endl; // qStdout() << settings.keyValueserialize(propKey) << endl;
// return 0; // return 0;
// } // }
@@ -386,7 +386,7 @@ int CliApplication::exec() {
QString noteFilePath = QString("%1/%2.txt").arg(paths::noteDraftsDir()).arg(note.idString()); QString noteFilePath = QString("%1/%2.txt").arg(paths::noteDraftsDir()).arg(note.idString());
if (!filePutContents(noteFilePath, note.toFriendlyString())) { if (!filePutContents(noteFilePath, note.serialize())) {
qStderr() << QString("Cannot open %1 for writing").arg(noteFilePath) << endl; qStderr() << QString("Cannot open %1 for writing").arg(noteFilePath) << endl;
return 1; return 1;
} }
@@ -431,13 +431,13 @@ int CliApplication::exec() {
if (propKey.isEmpty()) { if (propKey.isEmpty()) {
QStringList propKeys = settings.allKeys(); QStringList propKeys = settings.allKeys();
for (int i = 0; i < propKeys.size(); i++) { for (int i = 0; i < propKeys.size(); i++) {
qStdout() << settings.keyValueToFriendlyString(propKeys[i]) << endl; qStdout() << settings.keyValueserialize(propKeys[i]) << endl;
} }
return 0; return 0;
} }
if (propValue.isEmpty()) { if (propValue.isEmpty()) {
qStdout() << settings.keyValueToFriendlyString(propKey) << endl; qStdout() << settings.keyValueserialize(propKey) << endl;
return 0; return 0;
} }

2
QtClient/JoplinQtClient/models/item.cpp
View File

@@ -5,7 +5,7 @@ namespace jop {
Item::Item() {} Item::Item() {}
QString Item::toFriendlyString() const { QString Item::serialize() const {
QStringList shownKeys; QStringList shownKeys;
shownKeys << "author" << "longitude" << "latitude" << "is_todo" << "todo_due" << "todo_completed"; shownKeys << "author" << "longitude" << "latitude" << "is_todo" << "todo_due" << "todo_completed";

2
QtClient/JoplinQtClient/models/item.h
View File

@@ -14,7 +14,7 @@ class Item : public BaseModel {
public: public:
Item(); Item();
QString toFriendlyString() const; QString serialize() const;
void patchFriendlyString(const QString& patch); void patchFriendlyString(const QString& patch);
}; };

2
QtClient/JoplinQtClient/settings.cpp
View File

@@ -35,6 +35,6 @@ int Settings::valueInt(const QString &name, int defaultValue) {
return value(name, defaultValue).toInt(); return value(name, defaultValue).toInt();
} }
QString Settings::keyValueToFriendlyString(const QString& key) const { QString Settings::keyValueserialize(const QString& key) const {
return QString("%1 = %2").arg(key).arg(value(key).toString()); return QString("%1 = %2").arg(key).arg(value(key).toString());
} }

2
QtClient/JoplinQtClient/settings.h
View File

@@ -15,7 +15,7 @@ public:
Settings(); Settings();
static void initialize(); static void initialize();
QString keyValueToFriendlyString(const QString& key) const; QString keyValueserialize(const QString& key) const;
public slots: public slots:

4
ReactNativeClient/src/base-model.js
View File

@@ -161,7 +161,7 @@ class BaseModel {
let itemId = o.id; let itemId = o.id;
if (options.autoTimestamp && this.hasField('updated_time')) { if (options.autoTimestamp && this.hasField('updated_time')) {
o.updated_time = time.unix(); o.updated_time = time.unixMs();
} }
if (options.isNew) { if (options.isNew) {
@@ -171,7 +171,7 @@ class BaseModel {
} }
if (!o.created_time && this.hasField('created_time')) { if (!o.created_time && this.hasField('created_time')) {
o.created_time = time.unix(); o.created_time = time.unixMs();
} }
query = Database.insertQuery(this.tableName(), o); query = Database.insertQuery(this.tableName(), o);

8
ReactNativeClient/src/file-api-driver-local.js
View File

@@ -32,10 +32,10 @@ class FileApiDriverLocal {
metadataFromStats_(path, stats) { metadataFromStats_(path, stats) {
return { return {
path: path, path: path,
createdTime: this.statTimeToUnixTimestamp_(stats.birthtime), created_time: this.statTimeToUnixTimestamp_(stats.birthtime),
updatedTime: this.statTimeToUnixTimestamp_(stats.mtime), updated_time: this.statTimeToUnixTimestamp_(stats.mtime),
createdTimeOrig: stats.birthtime, created_time_orig: stats.birthtime,
updatedTimeOrig: stats.mtime, updated_time_orig: stats.mtime,
isDir: stats.isDirectory(), isDir: stats.isDirectory(),
}; };
} }

10
ReactNativeClient/src/file-api-driver-memory.js
View File

@@ -23,12 +23,12 @@ class FileApiDriverMemory {
} }
newItem(path, isDir = false) { newItem(path, isDir = false) {
let now = time.unix(); let now = time.unixMs();
return { return {
path: path, path: path,
isDir: isDir, isDir: isDir,
updatedTime: now, updated_time: now, // In milliseconds!!
createdTime: now, created_time: now, // In milliseconds!!
content: '', content: '',
}; };
} }
@@ -41,7 +41,7 @@ class FileApiDriverMemory {
setTimestamp(path, timestamp) { setTimestamp(path, timestamp) {
let item = this.itemByPath(path); let item = this.itemByPath(path);
if (!item) return Promise.reject(new Error('File not found: ' + path)); if (!item) return Promise.reject(new Error('File not found: ' + path));
item.updatedTime = timestamp; item.updated_time = timestamp;
return Promise.resolve(); return Promise.resolve();
} }
@@ -85,7 +85,7 @@ class FileApiDriverMemory {
this.items_.push(item); this.items_.push(item);
} else { } else {
this.items_[index].content = content; this.items_[index].content = content;
this.items_[index].updatedTime = time.unix(); this.items_[index].updated_time = time.unix();
} }
return Promise.resolve(); return Promise.resolve();
} }

31
ReactNativeClient/src/file-api.js
View File

@@ -41,34 +41,6 @@ class FileApi {
return this.driver_.list(this.baseDir_).then((items) => { return this.driver_.list(this.baseDir_).then((items) => {
return this.scopeItemsToBaseDir_(items); return this.scopeItemsToBaseDir_(items);
}); });
// let fullPath = this.fullPath_(path);
// return this.driver_.list(fullPath).then((items) => {
// return items;
// // items = this.scopeItemsToBaseDir_(items);
// // if (recursive) {
// // let chain = [];
// // for (let i = 0; i < items.length; i++) {
// // let item = items[i];
// // if (!item.isDir) continue;
// // chain.push(() => {
// // return this.list(item.path, true).then((children) => {
// // for (let j = 0; j < children.length; j++) {
// // let md = children[j];
// // md.path = item.path + '/' + md.path;
// // items.push(md);
// // }
// // });
// // });
// // }
// // return promiseChain(chain).then(() => {
// // return items;
// // });
// // } else {
// // return items;
// // }
// });
} }
setTimestamp(path, timestamp) { setTimestamp(path, timestamp) {
@@ -81,7 +53,6 @@ class FileApi {
} }
stat(path) { stat(path) {
//console.info('stat ' + path);
return this.driver_.stat(this.fullPath_(path)).then((output) => { return this.driver_.stat(this.fullPath_(path)).then((output) => {
if (!output) return output; if (!output) return output;
output.path = path; output.path = path;
@@ -90,12 +61,10 @@ class FileApi {
} }
get(path) { get(path) {
//console.info('get ' + path);
return this.driver_.get(this.fullPath_(path)); return this.driver_.get(this.fullPath_(path));
} }
put(path, content) { put(path, content) {
//console.info('put ' + path);
return this.driver_.put(this.fullPath_(path), content); return this.driver_.put(this.fullPath_(path), content);
} }

50
ReactNativeClient/src/models/base-item.js
View File

@@ -3,6 +3,7 @@ import { Note } from 'src/models/note.js';
import { Folder } from 'src/models/folder.js'; import { Folder } from 'src/models/folder.js';
import { folderItemFilename } from 'src/string-utils.js' import { folderItemFilename } from 'src/string-utils.js'
import { Database } from 'src/database.js'; import { Database } from 'src/database.js';
import { time } from 'src/time-utils.js';
import moment from 'moment'; import moment from 'moment';
class BaseItem extends BaseModel { class BaseItem extends BaseModel {
@@ -17,8 +18,15 @@ class BaseItem extends BaseModel {
static itemClass(item) { static itemClass(item) {
if (!item) throw new Error('Item cannot be null'); if (!item) throw new Error('Item cannot be null');
if (!('type_' in item)) throw new Error('Item does not have a type_ property');
return item.type_ == BaseModel.ITEM_TYPE_NOTE ? Note : Folder; if (typeof item === 'object') {
if (!('type_' in item)) throw new Error('Item does not have a type_ property');
return item.type_ == BaseModel.ITEM_TYPE_NOTE ? Note : Folder;
} else {
if (Number(item) === BaseModel.ITEM_TYPE_NOTE) return Note;
if (Number(item) === BaseModel.ITEM_TYPE_FOLDER) return Folder;
throw new Error('Unknown type: ' + item);
}
} }
static pathToId(path) { static pathToId(path) {
@@ -34,10 +42,10 @@ class BaseItem extends BaseModel {
}); });
} }
static toFriendlyString_format(propName, propValue) { static serialize_format(propName, propValue) {
if (['created_time', 'updated_time'].indexOf(propName) >= 0) { if (['created_time', 'updated_time'].indexOf(propName) >= 0) {
if (!propValue) return ''; if (!propValue) return '';
propValue = moment.unix(propValue).utc().format('YYYY-MM-DD HH:mm:ss') + 'Z'; propValue = moment.unix(propValue / 1000).utc().format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z';
} else if (propValue === null || propValue === undefined) { } else if (propValue === null || propValue === undefined) {
propValue = ''; propValue = '';
} }
@@ -45,20 +53,22 @@ class BaseItem extends BaseModel {
return propValue; return propValue;
} }
static fromFriendlyString_format(propName, propValue) { static unserialize_format(type, propName, propValue) {
if (propName == 'type_') return propValue; if (propName == 'type_') return propValue;
let ItemClass = this.itemClass(type);
if (['created_time', 'updated_time'].indexOf(propName) >= 0) { if (['created_time', 'updated_time'].indexOf(propName) >= 0) {
if (!propValue) return 0; if (!propValue) return 0;
propValue = moment(propValue, 'YYYY-MM-DD HH:mm:ssZ').unix(); propValue = moment(propValue, 'YYYY-MM-DDTHH:mm:ss.SSSZ').format('x');
} else { } else {
propValue = Database.formatValue(this.fieldType(propName), propValue); propValue = Database.formatValue(ItemClass.fieldType(propName), propValue);
} }
return propValue; return propValue;
} }
static toFriendlyString(item, type = null, shownKeys = null) { static serialize(item, type = null, shownKeys = null) {
let output = []; let output = [];
output.push(item.title); output.push(item.title);
@@ -67,14 +77,14 @@ class BaseItem extends BaseModel {
output.push(''); output.push('');
for (let i = 0; i < shownKeys.length; i++) { for (let i = 0; i < shownKeys.length; i++) {
let v = item[shownKeys[i]]; let v = item[shownKeys[i]];
v = this.toFriendlyString_format(shownKeys[i], v); v = this.serialize_format(shownKeys[i], v);
output.push(shownKeys[i] + ': ' + v); output.push(shownKeys[i] + ': ' + v);
} }
return output.join("\n"); return output.join("\n");
} }
static fromFriendlyString(content) { static unserialize(content) {
let lines = content.split("\n"); let lines = content.split("\n");
let output = {}; let output = {};
let state = 'readingProps'; let state = 'readingProps';
@@ -94,7 +104,7 @@ class BaseItem extends BaseModel {
if (p < 0) throw new Error('Invalid property format: ' + line + ": " + content); if (p < 0) throw new Error('Invalid property format: ' + line + ": " + content);
let key = line.substr(0, p).trim(); let key = line.substr(0, p).trim();
let value = line.substr(p + 1).trim(); let value = line.substr(p + 1).trim();
output[key] = this.fromFriendlyString_format(key, value); output[key] = value;
} else if (state == 'readingBody') { } else if (state == 'readingBody') {
body.splice(0, 0, line); body.splice(0, 0, line);
} }
@@ -104,11 +114,29 @@ class BaseItem extends BaseModel {
let title = body.splice(0, 2); let title = body.splice(0, 2);
output.title = title[0]; output.title = title[0];
if (!output.type_) throw new Error('Missing required property: type_: ' + content);
output.type_ = Number(output.type_);
if (output.type_ == BaseModel.ITEM_TYPE_NOTE) output.body = body.join("\n"); if (output.type_ == BaseModel.ITEM_TYPE_NOTE) output.body = body.join("\n");
for (let n in output) {
if (!output.hasOwnProperty(n)) continue;
output[n] = this.unserialize_format(output.type_, n, output[n]);
}
return output; return output;
} }
static itemsThatNeedSync(limit = 100) {
return Folder.modelSelectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit).then((items) => {
if (items.length) return { hasMore: true, items: items };
return Note.modelSelectAll('SELECT * FROM notes WHERE sync_time < updated_time LIMIT ' + limit).then((items) => {
return { hasMore: items.length >= limit, items: items };
});
});
}
} }
export { BaseItem }; export { BaseItem };

4
ReactNativeClient/src/models/folder.js
View File

@@ -13,8 +13,8 @@ class Folder extends BaseItem {
return 'folders'; return 'folders';
} }
static toFriendlyString(folder) { static serialize(folder) {
return super.toFriendlyString(folder, 'folder', ['id', 'created_time', 'updated_time', 'type_']); return super.serialize(folder, 'folder', ['id', 'created_time', 'updated_time', 'type_']);
} }
static itemType() { static itemType() {

4
ReactNativeClient/src/models/note.js
View File

@@ -12,8 +12,8 @@ class Note extends BaseItem {
return 'notes'; return 'notes';
} }
static toFriendlyString(note, type = null, shownKeys = null) { static serialize(note, type = null, shownKeys = null) {
return super.toFriendlyString(note, 'note', ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time', 'id', 'parent_id', 'type_']); return super.serialize(note, 'note', ["author", "longitude", "latitude", "is_todo", "todo_due", "todo_completed", 'created_time', 'updated_time', 'id', 'parent_id', 'type_']);
} }
static itemType() { static itemType() {

466
ReactNativeClient/src/synchronizer.js
View File

@@ -1,34 +1,17 @@
require('babel-plugin-transform-runtime'); require('babel-plugin-transform-runtime');
import { Log } from 'src/log.js';
import { Setting } from 'src/models/setting.js';
import { Change } from 'src/models/change.js';
import { Folder } from 'src/models/folder.js';
import { Note } from 'src/models/note.js';
import { BaseItem } from 'src/models/base-item.js'; import { BaseItem } from 'src/models/base-item.js';
import { BaseModel } from 'src/base-model.js';
import { promiseChain } from 'src/promise-utils.js';
import { NoteFolderService } from 'src/services/note-folder-service.js';
import { time } from 'src/time-utils.js';
import { sprintf } from 'sprintf-js'; import { sprintf } from 'sprintf-js';
//import { promiseWhile } from 'src/promise-utils.js'; import { time } from 'src/time-utils.js';
import moment from 'moment'; import { Log } from 'src/log.js'
const fs = require('fs');
const path = require('path');
class Synchronizer { class Synchronizer {
constructor(db, api) { constructor(db, api) {
this.state_ = 'idle';
this.db_ = db; this.db_ = db;
this.api_ = api; this.api_ = api;
} }
state() {
return this.state_;
}
db() { db() {
return this.db_; return this.db_;
} }
@@ -37,382 +20,101 @@ class Synchronizer {
return this.api_; return this.api_;
} }
loadParentAndItem(change) { async start() {
if (change.item_type == BaseModel.ITEM_TYPE_NOTE) { // ------------------------------------------------------------------------
return Note.load(change.item_id).then((note) => { // First, find all the items that have been changed since the
if (!note) return { parent:null, item: null }; // last sync and apply the changes to remote.
// ------------------------------------------------------------------------
return Folder.load(note.parent_id).then((folder) => {
return Promise.resolve({ parent: folder, item: note });
});
});
} else {
return Folder.load(change.item_id).then((folder) => {
return Promise.resolve({ parent: null, item: folder });
});
}
}
remoteFileByPath(remoteFiles, path) {
for (let i = 0; i < remoteFiles.length; i++) {
if (remoteFiles[i].path == path) return remoteFiles[i];
}
return null;
}
conflictDir(remoteFiles) {
let d = this.remoteFileByPath('Conflicts');
if (!d) {
return this.api().mkdir('Conflicts').then(() => {
return 'Conflicts';
});
} else {
return Promise.resolve('Conflicts');
}
}
moveConflict(item) {
// No need to handle folder conflicts
if (item.type == 'folder') return Promise.resolve();
return this.conflictDir().then((conflictDirPath) => {
let p = path.basename(item.path).split('.');
let pos = item.type == 'folder' ? p.length - 1 : p.length - 2;
p.splice(pos, 0, moment().format('YYYYMMDDThhmmss'));
let newPath = p.join('.');
return this.api().move(item.path, conflictDirPath + '/' + newPath);
});
}
itemByPath(items, path) {
for (let i = 0; i < items.length; i++) {
if (items[i].path == path) return items[i];
}
return null;
}
itemIsSameDate(item, date) {
return item.updatedTime === date;
}
itemIsStrictlyNewerThan(item, date) {
return item.updatedTime > date;
}
itemIsStrictlyOlderThan(item, date) {
return item.updatedTime < date;
}
dbItemToSyncItem(dbItem) {
if (!dbItem) return null;
return {
type: dbItem.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
path: Folder.systemPath(dbItem),
syncTime: dbItem.sync_time,
updatedTime: dbItem.updated_time,
dbItem: dbItem,
};
}
remoteItemToSyncItem(remoteItem) {
if (!remoteItem) return null;
return {
type: remoteItem.content.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
path: remoteItem.path,
syncTime: 0,
updatedTime: remoteItem.updatedTime,
remoteItem: remoteItem,
};
}
syncAction(localItem, remoteItem, deletedLocalPaths) {
let output = this.syncActions(localItem ? [localItem] : [], remoteItem ? [remoteItem] : [], deletedLocalPaths);
if (output.length > 1) throw new Error('Invalid number of actions returned');
return output.length ? output[0] : null;
}
// Assumption: it's not possible to, for example, have a directory one the dest
// and a file with the same name on the source. It's not possible because the
// file and directory names are UUID so should be unique.
// Each item must have these properties:
// - path
// - type
// - syncTime
// - updatedTime
syncActions(localItems, remoteItems, deletedLocalPaths) {
let output = [];
let donePaths = []; let donePaths = [];
// console.info('==================================================');
// console.info(localItems, remoteItems);
for (let i = 0; i < localItems.length; i++) {
let local = localItems[i];
let remote = this.itemByPath(remoteItems, local.path);
let action = {
local: local,
remote: remote,
};
if (!remote) {
if (local.syncTime) {
action.type = 'delete';
action.dest = 'local';
action.reason = 'Local has been synced to remote previously, but remote no longer exist, which means remote has been deleted';
} else {
action.type = 'create';
action.dest = 'remote';
action.reason = 'Local has never been synced to remote, and remote does not exists, which means remote must be created';
}
} else {
if (this.itemIsStrictlyOlderThan(local, local.syncTime)) continue;
if (this.itemIsStrictlyOlderThan(remote, local.updatedTime)) {
action.type = 'update';
action.dest = 'remote';
action.reason = sprintf('Remote (%s) was modified before updated time of local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString(),);
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && this.itemIsStrictlyNewerThan(local, local.syncTime)) {
action.type = 'conflict';
action.reason = sprintf('Both remote (%s) and local (%s) were modified after the last sync (%s).',
moment.unix(remote.updatedTime).toISOString(),
moment.unix(local.updatedTime).toISOString(),
moment.unix(local.syncTime).toISOString()
);
if (local.type == 'folder') {
action.solution = [
{ type: 'update', dest: 'local' },
];
} else {
action.solution = [
{ type: 'copy-to-remote-conflict-dir', dest: 'local' },
{ type: 'copy-to-local-conflict-dir', dest: 'local' },
{ type: 'update', dest: 'local' },
];
}
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && local.updatedTime <= local.syncTime) {
action.type = 'update';
action.dest = 'local';
action.reason = sprintf('Remote (%s) was modified after update time of local (%s). And sync time (%s) is the same or more recent than local update time', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString());
} else {
continue; // Neither local nor remote item have been changed recently
}
}
donePaths.push(local.path);
output.push(action);
}
for (let i = 0; i < remoteItems.length; i++) {
let remote = remoteItems[i];
if (donePaths.indexOf(remote.path) >= 0) continue; // Already handled in the previous loop
let local = this.itemByPath(localItems, remote.path);
let action = {
local: local,
remote: remote,
};
if (!local) {
if (deletedLocalPaths.indexOf(remote.path) >= 0) {
action.type = 'delete';
action.dest = 'remote';
} else {
action.type = 'create';
action.dest = 'local';
}
} else {
if (this.itemIsStrictlyOlderThan(remote, local.syncTime)) continue; // Already have this version
// Note: no conflict is possible here since if the local item has been
// modified since the last sync, it's been processed in the previous loop.
// So throw an exception is this normally impossible condition happens anyway.
// It's handled at condition this.itemIsStrictlyNewerThan(remote, local.syncTime) in above loop
if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) {
console.error('Remote cannot be newer than last sync time', remote, local);
throw new Error('Remote cannot be newer than last sync time');
}
if (this.itemIsStrictlyNewerThan(remote, local.updatedTime)) {
action.type = 'update';
action.dest = 'local';
action.reason = sprintf('Remote (%s) was modified after local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.updatedTime).toISOString(),);;
} else {
continue;
}
}
output.push(action);
}
// console.info('-----------------------------------------');
// console.info(output);
return output;
}
processState(state) {
Log.info('Sync: processing: ' + state);
this.state_ = state;
if (state == 'uploadChanges') {
return this.processState_uploadChanges();
} else if (state == 'downloadChanges') {
//return this.processState('idle');
return this.processState_downloadChanges();
} else if (state == 'idle') {
// Nothing
return Promise.resolve();
} else {
throw new Error('Invalid state: ' . state);
}
}
processSyncAction(action) {
//console.info('Sync action: ', action);
//console.info('Sync action: ' + JSON.stringify(action));
if (!action) return Promise.resolve();
console.info('Sync action: ' + action.type + ' ' + action.dest + ': ' + action.reason);
if (action.type == 'conflict') {
console.info(action);
} else {
let syncItem = action[action.dest == 'local' ? 'remote' : 'local'];
let path = syncItem.path;
if (action.type == 'create') {
if (action.dest == 'remote') {
let content = null;
let dbItem = syncItem.dbItem;
if (syncItem.type == 'folder') {
content = Folder.toFriendlyString(dbItem);
} else {
content = Note.toFriendlyString(dbItem);
}
return this.api().put(path, content).then(() => {
return this.api().setTimestamp(path, dbItem.updated_time);
});
// TODO: save sync_time
} else {
let dbItem = syncItem.remoteItem.content;
dbItem.sync_time = time.unix();
dbItem.updated_time = action.remote.updatedTime;
if (syncItem.type == 'folder') {
return Folder.save(dbItem, { isNew: true, autoTimestamp: false });
} else {
return Note.save(dbItem, { isNew: true, autoTimestamp: false });
}
// TODO: save sync_time
}
}
if (action.type == 'update') {
if (action.dest == 'remote') {
let dbItem = syncItem.dbItem;
let ItemClass = BaseItem.itemClass(dbItem);
let content = ItemClass.toFriendlyString(dbItem);
//console.info('PUT', content);
return this.api().put(path, content).then(() => {
return this.api().setTimestamp(path, dbItem.updated_time);
}).then(() => {
let toSave = { id: dbItem.id, sync_time: time.unix() };
return NoteFolderService.save(syncItem.type, dbItem, null, { autoTimestamp: false });
});
} else {
let dbItem = Object.assign({}, syncItem.remoteItem.content);
dbItem.sync_time = time.unix();
return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem, { autoTimestamp: false });
}
}
}
return Promise.resolve(); // TODO
}
async processLocalItem(dbItem) {
let localItem = this.dbItemToSyncItem(dbItem);
let remoteItem = await this.api().stat(localItem.path);
let action = this.syncAction(localItem, remoteItem, []);
await this.processSyncAction(action);
let toSave = Object.assign({}, dbItem);
toSave.sync_time = time.unix();
return NoteFolderService.save(localItem.type, toSave, dbItem, { autoTimestamp: false });
}
async processRemoteItem(remoteItem) {
let content = await this.api().get(remoteItem.path);
if (!content) throw new Error('Cannot get content for: ' + remoteItem.path);
remoteItem.content = Note.fromFriendlyString(content);
let remoteSyncItem = this.remoteItemToSyncItem(remoteItem);
let dbItem = await BaseItem.loadItemByPath(remoteItem.path);
let localSyncItem = this.dbItemToSyncItem(dbItem);
let action = this.syncAction(localSyncItem, remoteSyncItem, []);
return this.processSyncAction(action);
}
async processState_uploadChanges() {
while (true) { while (true) {
let result = await NoteFolderService.itemsThatNeedSync(50); let result = await BaseItem.itemsThatNeedSync();
console.info('Items that need sync: ' + result.items.length); let locals = result.items;
for (let i = 0; i < result.items.length; i++) {
let item = result.items[i]; for (let i = 0; i < locals.length; i++) {
await this.processLocalItem(item); let local = locals[i];
let ItemClass = BaseItem.itemClass(local);
let path = BaseItem.systemPath(local);
// Safety check to avoid infinite loops:
if (donePaths.indexOf(path) > 0) throw new Error(sprintf('Processing a path that has already been done: %s. sync_time was not updated?', path));
let remote = await this.api().stat(path);
let content = ItemClass.serialize(local);
let action = null;
if (!remote) {
action = 'createRemote';
} else {
if (remote.updated_time > local.updated_time) {
action = 'conflict';
} else {
action = 'updateRemote';
}
}
if (action == 'createRemote' || action == 'updateRemote') {
await this.api().put(path, content);
await this.api().setTimestamp(path, local.updated_time);
} else if (action == 'conflict') {
console.warn('FOUND CONFLICT', local, remote);
}
let newLocal = { id: local.id, sync_time: time.unixMs(), type_: local.type_ };
await ItemClass.save(newLocal, { autoTimestamp: false });
donePaths.push(path);
} }
if (!result.hasMore) break; if (!result.hasMore) break;
} }
//console.info('DOWNLOAD CHANGE DISABLED'); return Promise.resolve(); // ------------------------------------------------------------------------
// Then, loop through all the remote items, find those that
return this.processState('downloadChanges'); // have been updated, and apply the changes to local.
} // ------------------------------------------------------------------------
async processState_downloadChanges() { // At this point all the local items that have changed have been pushed to remote
let items = await this.api().list(); // or handled as conflicts, so no conflict is possible after this.
for (let i = 0; i < items.length; i++) {
await this.processRemoteItem(items[i]); let remotes = await this.api().list();
for (let i = 0; i < remotes.length; i++) {
let remote = remotes[i];
let path = remote.path;
if (donePaths.indexOf(path) > 0) continue;
let action = null;
let local = await BaseItem.loadItemByPath(path);
if (!local) {
action = 'createLocal';
} else {
if (remote.updated_time > local.updated_time) {
action = 'updateLocal';
}
}
if (!action) continue;
if (action == 'createLocal' || action == 'updateLocal') {
let content = await this.api().get(path);
if (!content) {
Log.warn('Remote item has been deleted between now and the list() call? In that case it will handled during the next sync: ' + path);
continue;
}
content = BaseItem.unserialize(content);
let ItemClass = BaseItem.itemClass(content);
content.sync_time = time.unixMs();
let options = { autoTimestamp: false };
if (action == 'createLocal') options.isNew = true;
await ItemClass.save(content, options);
}
} }
return this.processState('idle');
return Promise.resolve();
} }
start() {
Log.info('Sync: start');
if (this.state() != 'idle') {
return Promise.reject('Cannot start synchronizer because synchronization already in progress. State: ' + this.state());
}
this.state_ = 'started';
// if (!this.api().session()) {
// Log.info("Sync: cannot start synchronizer because user is not logged in.");
// return;
// }
return this.processState('uploadChanges').catch((error) => {
console.info('Synchronizer error:', error);
throw error;
});
}
} }
export { Synchronizer }; export { Synchronizer };

6
ReactNativeClient/src/synchronizer_old.js
View File

@@ -65,7 +65,7 @@ class Synchronizer {
} else if (c.type == Change.TYPE_CREATE) { } else if (c.type == Change.TYPE_CREATE) {
p = this.loadParentAndItem(c).then((result) => { p = this.loadParentAndItem(c).then((result) => {
let options = { let options = {
contents: Note.toFriendlyString(result.item), contents: Note.serialize(result.item),
path: Note.systemPath(result.parent, result.item), path: Note.systemPath(result.parent, result.item),
mode: 'overwrite', mode: 'overwrite',
// client_modified: // client_modified:
@@ -79,7 +79,7 @@ class Synchronizer {
// console.info(item); // console.info(item);
// let options = { // let options = {
// contents: Note.toFriendlyString(item), // contents: Note.serialize(item),
// path: Note.systemPath(item), // path: Note.systemPath(item),
// mode: 'overwrite', // mode: 'overwrite',
// // client_modified: // // client_modified:
@@ -87,7 +87,7 @@ class Synchronizer {
// // console.info(options); // // console.info(options);
// //let content = Note.toFriendlyString(item); // //let content = Note.serialize(item);
// //console.info(content); // //console.info(content);
// //console.info('SYNC', item); // //console.info('SYNC', item);

418
ReactNativeClient/src/synchronizer_old2.js Normal file
View File

@@ -0,0 +1,418 @@
require('babel-plugin-transform-runtime');
import { Log } from 'src/log.js';
import { Setting } from 'src/models/setting.js';
import { Change } from 'src/models/change.js';
import { Folder } from 'src/models/folder.js';
import { Note } from 'src/models/note.js';
import { BaseItem } from 'src/models/base-item.js';
import { BaseModel } from 'src/base-model.js';
import { promiseChain } from 'src/promise-utils.js';
import { NoteFolderService } from 'src/services/note-folder-service.js';
import { time } from 'src/time-utils.js';
import { sprintf } from 'sprintf-js';
//import { promiseWhile } from 'src/promise-utils.js';
import moment from 'moment';
const fs = require('fs');
const path = require('path');
class Synchronizer {
constructor(db, api) {
this.state_ = 'idle';
this.db_ = db;
this.api_ = api;
}
state() {
return this.state_;
}
db() {
return this.db_;
}
api() {
return this.api_;
}
loadParentAndItem(change) {
if (change.item_type == BaseModel.ITEM_TYPE_NOTE) {
return Note.load(change.item_id).then((note) => {
if (!note) return { parent:null, item: null };
return Folder.load(note.parent_id).then((folder) => {
return Promise.resolve({ parent: folder, item: note });
});
});
} else {
return Folder.load(change.item_id).then((folder) => {
return Promise.resolve({ parent: null, item: folder });
});
}
}
remoteFileByPath(remoteFiles, path) {
for (let i = 0; i < remoteFiles.length; i++) {
if (remoteFiles[i].path == path) return remoteFiles[i];
}
return null;
}
conflictDir(remoteFiles) {
let d = this.remoteFileByPath('Conflicts');
if (!d) {
return this.api().mkdir('Conflicts').then(() => {
return 'Conflicts';
});
} else {
return Promise.resolve('Conflicts');
}
}
moveConflict(item) {
// No need to handle folder conflicts
if (item.type == 'folder') return Promise.resolve();
return this.conflictDir().then((conflictDirPath) => {
let p = path.basename(item.path).split('.');
let pos = item.type == 'folder' ? p.length - 1 : p.length - 2;
p.splice(pos, 0, moment().format('YYYYMMDDThhmmss'));
let newPath = p.join('.');
return this.api().move(item.path, conflictDirPath + '/' + newPath);
});
}
itemByPath(items, path) {
for (let i = 0; i < items.length; i++) {
if (items[i].path == path) return items[i];
}
return null;
}
itemIsSameDate(item, date) {
return item.updated_time === date;
}
itemIsStrictlyNewerThan(item, date) {
return item.updated_time > date;
}
itemIsStrictlyOlderThan(item, date) {
return item.updated_time < date;
}
dbItemToSyncItem(dbItem) {
if (!dbItem) return null;
return {
type: dbItem.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
path: Folder.systemPath(dbItem),
syncTime: dbItem.sync_time,
updated_time: dbItem.updated_time,
dbItem: dbItem,
};
}
remoteItemToSyncItem(remoteItem) {
if (!remoteItem) return null;
return {
type: remoteItem.content.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
path: remoteItem.path,
syncTime: 0,
updated_time: remoteItem.updated_time,
remoteItem: remoteItem,
};
}
syncAction(localItem, remoteItem, deletedLocalPaths) {
let output = this.syncActions(localItem ? [localItem] : [], remoteItem ? [remoteItem] : [], deletedLocalPaths);
if (output.length > 1) throw new Error('Invalid number of actions returned');
return output.length ? output[0] : null;
}
// Assumption: it's not possible to, for example, have a directory one the dest
// and a file with the same name on the source. It's not possible because the
// file and directory names are UUID so should be unique.
// Each item must have these properties:
// - path
// - type
// - syncTime
// - updated_time
syncActions(localItems, remoteItems, deletedLocalPaths) {
let output = [];
let donePaths = [];
// console.info('==================================================');
// console.info(localItems, remoteItems);
for (let i = 0; i < localItems.length; i++) {
let local = localItems[i];
let remote = this.itemByPath(remoteItems, local.path);
let action = {
local: local,
remote: remote,
};
if (!remote) {
if (local.syncTime) {
action.type = 'delete';
action.dest = 'local';
action.reason = 'Local has been synced to remote previously, but remote no longer exist, which means remote has been deleted';
} else {
action.type = 'create';
action.dest = 'remote';
action.reason = 'Local has never been synced to remote, and remote does not exists, which means remote must be created';
}
} else {
if (this.itemIsStrictlyOlderThan(local, local.syncTime)) continue;
if (this.itemIsStrictlyOlderThan(remote, local.updated_time)) {
action.type = 'update';
action.dest = 'remote';
action.reason = sprintf('Remote (%s) was modified before updated time of local (%s).', moment.unix(remote.updated_time).toISOString(), moment.unix(local.syncTime).toISOString(),);
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && this.itemIsStrictlyNewerThan(local, local.syncTime)) {
action.type = 'conflict';
action.reason = sprintf('Both remote (%s) and local (%s) were modified after the last sync (%s).',
moment.unix(remote.updated_time).toISOString(),
moment.unix(local.updated_time).toISOString(),
moment.unix(local.syncTime).toISOString()
);
if (local.type == 'folder') {
action.solution = [
{ type: 'update', dest: 'local' },
];
} else {
action.solution = [
{ type: 'copy-to-remote-conflict-dir', dest: 'local' },
{ type: 'copy-to-local-conflict-dir', dest: 'local' },
{ type: 'update', dest: 'local' },
];
}
} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && local.updated_time <= local.syncTime) {
action.type = 'update';
action.dest = 'local';
action.reason = sprintf('Remote (%s) was modified after update time of local (%s). And sync time (%s) is the same or more recent than local update time', moment.unix(remote.updated_time).toISOString(), moment.unix(local.updated_time).toISOString(), moment.unix(local.syncTime).toISOString());
} else {
continue; // Neither local nor remote item have been changed recently
}
}
donePaths.push(local.path);
output.push(action);
}
for (let i = 0; i < remoteItems.length; i++) {
let remote = remoteItems[i];
if (donePaths.indexOf(remote.path) >= 0) continue; // Already handled in the previous loop
let local = this.itemByPath(localItems, remote.path);
let action = {
local: local,
remote: remote,
};
if (!local) {
if (deletedLocalPaths.indexOf(remote.path) >= 0) {
action.type = 'delete';
action.dest = 'remote';
} else {
action.type = 'create';
action.dest = 'local';
}
} else {
if (this.itemIsStrictlyOlderThan(remote, local.syncTime)) continue; // Already have this version
// Note: no conflict is possible here since if the local item has been
// modified since the last sync, it's been processed in the previous loop.
// So throw an exception is this normally impossible condition happens anyway.
// It's handled at condition this.itemIsStrictlyNewerThan(remote, local.syncTime) in above loop
if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) {
console.error('Remote cannot be newer than last sync time', remote, local);
throw new Error('Remote cannot be newer than last sync time');
}
if (this.itemIsStrictlyNewerThan(remote, local.updated_time)) {
action.type = 'update';
action.dest = 'local';
action.reason = sprintf('Remote (%s) was modified after local (%s).', moment.unix(remote.updated_time).toISOString(), moment.unix(local.updated_time).toISOString(),);;
} else {
continue;
}
}
output.push(action);
}
// console.info('-----------------------------------------');
// console.info(output);
return output;
}
processState(state) {
Log.info('Sync: processing: ' + state);
this.state_ = state;
if (state == 'uploadChanges') {
return this.processState_uploadChanges();
} else if (state == 'downloadChanges') {
//return this.processState('idle');
return this.processState_downloadChanges();
} else if (state == 'idle') {
// Nothing
return Promise.resolve();
} else {
throw new Error('Invalid state: ' . state);
}
}
processSyncAction(action) {
//console.info('Sync action: ', action);
//console.info('Sync action: ' + JSON.stringify(action));
if (!action) return Promise.resolve();
console.info('Sync action: ' + action.type + ' ' + action.dest + ': ' + action.reason);
if (action.type == 'conflict') {
console.info(action);
} else {
let syncItem = action[action.dest == 'local' ? 'remote' : 'local'];
let path = syncItem.path;
if (action.type == 'create') {
if (action.dest == 'remote') {
let content = null;
let dbItem = syncItem.dbItem;
if (syncItem.type == 'folder') {
content = Folder.serialize(dbItem);
} else {
content = Note.serialize(dbItem);
}
return this.api().put(path, content).then(() => {
return this.api().setTimestamp(path, dbItem.updated_time);
});
// TODO: save sync_time
} else {
let dbItem = syncItem.remoteItem.content;
dbItem.sync_time = time.unix();
dbItem.updated_time = action.remote.updated_time;
if (syncItem.type == 'folder') {
return Folder.save(dbItem, { isNew: true, autoTimestamp: false });
} else {
return Note.save(dbItem, { isNew: true, autoTimestamp: false });
}
// TODO: save sync_time
}
}
if (action.type == 'update') {
if (action.dest == 'remote') {
let dbItem = syncItem.dbItem;
let ItemClass = BaseItem.itemClass(dbItem);
let content = ItemClass.serialize(dbItem);
//console.info('PUT', content);
return this.api().put(path, content).then(() => {
return this.api().setTimestamp(path, dbItem.updated_time);
}).then(() => {
let toSave = { id: dbItem.id, sync_time: time.unix() };
return NoteFolderService.save(syncItem.type, dbItem, null, { autoTimestamp: false });
});
} else {
let dbItem = Object.assign({}, syncItem.remoteItem.content);
dbItem.sync_time = time.unix();
return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem, { autoTimestamp: false });
}
}
}
return Promise.resolve(); // TODO
}
async processLocalItem(dbItem) {
let localItem = this.dbItemToSyncItem(dbItem);
let remoteItem = await this.api().stat(localItem.path);
let action = this.syncAction(localItem, remoteItem, []);
await this.processSyncAction(action);
let toSave = Object.assign({}, dbItem);
toSave.sync_time = time.unix();
return NoteFolderService.save(localItem.type, toSave, dbItem, { autoTimestamp: false });
}
async processRemoteItem(remoteItem) {
let content = await this.api().get(remoteItem.path);
if (!content) throw new Error('Cannot get content for: ' + remoteItem.path);
remoteItem.content = Note.unserialize(content);
let remoteSyncItem = this.remoteItemToSyncItem(remoteItem);
let dbItem = await BaseItem.loadItemByPath(remoteItem.path);
let localSyncItem = this.dbItemToSyncItem(dbItem);
let action = this.syncAction(localSyncItem, remoteSyncItem, []);
return this.processSyncAction(action);
}
async processState_uploadChanges() {
while (true) {
let result = await NoteFolderService.itemsThatNeedSync(50);
console.info('Items that need sync: ' + result.items.length);
for (let i = 0; i < result.items.length; i++) {
let item = result.items[i];
await this.processLocalItem(item);
}
if (!result.hasMore) break;
}
//console.info('DOWNLOAD CHANGE DISABLED'); return Promise.resolve();
return this.processState('downloadChanges');
}
async processState_downloadChanges() {
let items = await this.api().list();
for (let i = 0; i < items.length; i++) {
await this.processRemoteItem(items[i]);
}
return this.processState('idle');
}
start() {
Log.info('Sync: start');
if (this.state() != 'idle') {
return Promise.reject('Cannot start synchronizer because synchronization already in progress. State: ' + this.state());
}
this.state_ = 'started';
// if (!this.api().session()) {
// Log.info("Sync: cannot start synchronizer because user is not logged in.");
// return;
// }
return this.processState('uploadChanges').catch((error) => {
console.info('Synchronizer error:', error);
throw error;
});
}
}
export { Synchronizer };

8
ReactNativeClient/src/time-utils.js
View File

@@ -2,6 +2,14 @@ let time = {
unix() { unix() {
return Math.round((new Date()).getTime() / 1000); return Math.round((new Date()).getTime() / 1000);
},
unixMs() {
return (new Date()).getTime();
},
unixMsToS(ms) {
return Math.round(ms / 1000);
} }
} }