sync

CliClient/tests/synchronizer.js

@@ -1,32 +1,141 @@
 import { time } from 'src/time-utils.js';
-import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi } from 'test-utils.js';
+import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient } from 'test-utils.js';
 import { createFoldersAndNotes } from 'test-data.js';
 import { Folder } from 'src/models/folder.js';
 import { Note } from 'src/models/note.js';
 import { BaseItem } from 'src/models/base-item.js';
+import { BaseModel } from 'src/base-model.js';
+
+async function localItemsSameAsRemote(locals, expect) {
+	try {
+		for (let i = 0; i < locals.length; i++) {
+			let dbItem = locals[i];
+			let path = BaseItem.systemPath(dbItem);
+			let remote = await fileApi().stat(path);
+
+			// console.info('=======================');
+			// console.info(remote);
+			// console.info(dbItem);
+			// console.info('=======================');
+
+			expect(!!remote).toBe(true);
+			expect(remote.updatedTime).toBe(dbItem.updated_time);
+
+			let remoteContent = await fileApi().get(path);
+			remoteContent = dbItem.type_ == BaseModel.ITEM_TYPE_NOTE ? Note.fromFriendlyString(remoteContent) : Folder.fromFriendlyString(remoteContent);
+			expect(remoteContent.title).toBe(dbItem.title);
+		}
+	} catch (error) {
+		console.error(error);
+	}
+}
 
 describe('Synchronizer', function() {
 
 	beforeEach( async (done) => {
-		await setupDatabaseAndSynchronizer();
+		await setupDatabaseAndSynchronizer(1);
+		await setupDatabaseAndSynchronizer(2);
+		switchClient(1);
 		done();
 	});
 
-	it('should create remote items', async (done) => {
+	// it('should create remote items', async (done) => {
-		let folder = await Folder.save({ title: "folder1" });
+	// 	let folder = await Folder.save({ title: "folder1" });
-		await Note.save({ title: "un", parent_id: folder.id });
+	// 	await Note.save({ title: "un", parent_id: folder.id });
+
+	// 	let all = await Folder.all(true);
+
+	// 	await synchronizer().start();
+
+	// 	await localItemsSameAsRemote(all, expect);
+
+	// 	done();
+	// });
+
+	// it('should update remote item', async (done) => {
+	// 	let folder = await Folder.save({ title: "folder1" });
+	// 	let note = await Note.save({ title: "un", parent_id: folder.id });
+
+	// 	await sleep(1);
+
+	// 	await Note.save({ title: "un UPDATE", id: note.id });
+
+	// 	let all = await Folder.all(true);
+	// 	await synchronizer().start();
+
+	// 	await localItemsSameAsRemote(all, expect);
+
+	// 	done();
+	// });
+
+	// it('should create local items', async (done) => {
+	// 	let folder = await Folder.save({ title: "folder1" });
+	// 	await Note.save({ title: "un", parent_id: folder.id });
+	// 	await synchronizer().start();
+	// 	await clearDatabase();
+	// 	await synchronizer().start();
+
+	// 	let all = await Folder.all(true);
+	// 	await localItemsSameAsRemote(all, expect);
+
+	// 	done();
+	// });
+
+	// it('should create same items on client 2', async (done) => {
+	// 	let folder = await Folder.save({ title: "folder1" });
+	// 	let note = await Note.save({ title: "un", parent_id: folder.id });
+	// 	await synchronizer().start();
+
+	// 	await sleep(1);
+
+	// 	switchClient(2);
+
+	// 	await synchronizer().start();
+
+	// 	let folder2 = await Folder.load(folder.id);
+	// 	let note2 = await Note.load(note.id);
+
+	// 	expect(!!folder2).toBe(true);
+	// 	expect(!!note2).toBe(true);
+
+	// 	expect(folder.title).toBe(folder.title);
+	// 	expect(folder.updated_time).toBe(folder.updated_time);
+
+	// 	expect(note.title).toBe(note.title);
+	// 	expect(note.updated_time).toBe(note.updated_time);
+	// 	expect(note.body).toBe(note.body);
+
+	// 	done();
+	// });
+
+	it('should update local items', async (done) => {
+		let folder1 = await Folder.save({ title: "folder1" });
+		let note1 = await Note.save({ title: "un", parent_id: folder1.id });
+		await synchronizer().start();
+
+		await sleep(1);
+
+		switchClient(2);
+
+		await synchronizer().start();
+
+		let note2 = await Note.load(note1.id);
+		note2.title = "Updated on client 2";
+		await Note.save(note2);
 
 		let all = await Folder.all(true);
 
 		await synchronizer().start();
 
-		for (let i = 0; i < all.length; i++) {
+		switchClient(1);
-			let dbItem = all[i];
+
-			let path = BaseItem.systemPath(all[i]);
+		await synchronizer().start();
-			let remote = await fileApi().stat(path);
+
-			expect(!!remote).toBe(true);
+		note1 = await Note.load(note1.id);
-			expect(remote.updatedTime).toBe(dbItem.updated_time);
+
-		}
+		expect(!!note1).toBe(true);
+		expect(note1.title).toBe(note2.title);
+		expect(note1.body).toBe(note2.body);
 
 		done();
 	});

CliClient/tests/test-utils.js

@@ -2,61 +2,95 @@ import fs from 'fs-extra';
 import { Database } from 'src/database.js';
 import { DatabaseDriverNode } from 'src/database-driver-node.js';
 import { BaseModel } from 'src/base-model.js';
+import { Folder } from 'src/models/folder.js';
+import { Note } from 'src/models/note.js';
+import { BaseItem } from 'src/models/base-item.js';
 import { Synchronizer } from 'src/synchronizer.js';
 import { FileApi } from 'src/file-api.js';
 import { FileApiDriverMemory } from 'src/file-api-driver-memory.js';
 
-let database_ = null;
+let databases_ = [];
-let synchronizer_ = null;
+let synchronizers_ = [];
 let fileApi_ = null;
+let currentClient_ = 1;
 
-function setupDatabase(done) {
+function sleep(n) {
-	if (database_) {
+	return new Promise((resolve, reject) => {
-		let queries = [
+		setTimeout(() => {
-			'DELETE FROM changes',
+			resolve();
-			'DELETE FROM notes',
+		}, n * 1000);
-			'DELETE FROM folders',
+	});
-			'DELETE FROM item_sync_times',
+}
-		];
 
-		return database_.transactionExecBatch(queries).then(() => {
+function switchClient(id) {
-			if (done) done();
+	currentClient_ = id;
-		});
+	BaseModel.db_ = databases_[id];
+	Folder.db_ = databases_[id];
+	Note.db_ = databases_[id];
+	BaseItem.db_ = databases_[id];
+}
+
+function clearDatabase(id = null) {
+	if (id === null) id = currentClient_;
+
+	let queries = [
+		'DELETE FROM changes',
+		'DELETE FROM notes',
+		'DELETE FROM folders',
+		'DELETE FROM item_sync_times',
+	];
+
+	return databases_[id].transactionExecBatch(queries);
+}
+
+function setupDatabase(id = null) {
+	if (id === null) id = currentClient_;
+
+	if (databases_[id]) {
+		return clearDatabase(id);
 	}
 
-	const filePath = __dirname + '/data/test.sqlite';
+	const filePath = __dirname + '/data/test-' + id + '.sqlite';
 	return fs.unlink(filePath).catch(() => {
 		// Don't care if the file doesn't exist
 	}).then(() => {
-		database_ = new Database(new DatabaseDriverNode());
+		databases_[id] = new Database(new DatabaseDriverNode());
-		database_.setDebugEnabled(false);
+		databases_[id].setDebugEnabled(false);
-		return database_.open({ name: filePath }).then(() => {
+		return databases_[id].open({ name: filePath }).then(() => {
-			BaseModel.db_ = database_;
+			BaseModel.db_ = databases_[id];
-			return setupDatabase(done);
+			return setupDatabase(id);
 		});
 	});
 }
 
-async function setupDatabaseAndSynchronizer() {
+async function setupDatabaseAndSynchronizer(id = null) {
-	await setupDatabase();
+	if (id === null) id = currentClient_;
 
-	if (!synchronizer_) {
+	await setupDatabase(id);
-		let fileDriver = new FileApiDriverMemory();
+
-		fileApi_ = new FileApi('/root', fileDriver);
+	if (!synchronizers_[id]) {
-		synchronizer_ = new Synchronizer(db(), fileApi_);
+		synchronizers_[id] = new Synchronizer(db(id), fileApi());
 	}
+
+	await fileApi().format();
 }
 
-function db() {
+function db(id = null) {
-	return database_;
+	if (id === null) id = currentClient_;
+	return databases_[id];
 }
 
-function synchronizer() {
+function synchronizer(id = null) {
-	return synchronizer_;
+	if (id === null) id = currentClient_;
+	console.info('SYNC', id);
+	return synchronizers_[id];
 }
 
 function fileApi() {
+	if (fileApi_) return fileApi_;
+
+	fileApi_ = new FileApi('/root', new FileApiDriverMemory());
 	return fileApi_;
 }
 
-export { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi };
+export { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient };

ReactNativeClient/src/models/folder.js

@@ -75,18 +75,11 @@ class Folder extends BaseItem {
 	}
 
 	static async all(includeNotes = false) {
-		let folders = await this.modelSelectAll('SELECT * FROM folders');
+		let folders = await Folder.modelSelectAll('SELECT * FROM folders');
 		if (!includeNotes) return folders;
 
-		let output = [];
+		let notes = await Note.modelSelectAll('SELECT * FROM notes');
-		for (let i = 0; i < folders.length; i++) {
+		return folders.concat(notes);
-			let folder = folders[i];
-			let notes = await Note.all(folder.id);
-			output.push(folder);
-			output = output.concat(notes);
-		}
-
-		return output;
 
 	}
 

ReactNativeClient/src/services/note-folder-service.js

@@ -32,8 +32,6 @@ class NoteFolderService extends BaseService {
 			toSave.id = item.id;
 		}
 
-		console.info(toSave);
-
 		return ItemClass.save(toSave, options).then((savedItem) => {
 			output = Object.assign(item, savedItem);
 			if (isNew && type == 'note') return Note.updateGeolocation(output.id);

ReactNativeClient/src/synchronizer.js

@@ -106,10 +106,8 @@ class Synchronizer {
 	dbItemToSyncItem(dbItem) {
 		if (!dbItem) return null;
 
-		let itemType = BaseModel.identifyItemType(dbItem);
-
 		return {
-			type: itemType == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
+			type: dbItem.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
 			path: Folder.systemPath(dbItem),
 			syncTime: dbItem.sync_time,
 			updatedTime: dbItem.updated_time,
@@ -121,7 +119,7 @@ class Synchronizer {
 		if (!remoteItem) return null;
 
 		return {
-			type: remoteItem.content.type,
+			type: remoteItem.content.type_ == BaseModel.ITEM_TYPE_FOLDER ? 'folder' : 'note',
 			path: remoteItem.path,
 			syncTime: 0,
 			updatedTime: remoteItem.updatedTime,
@@ -175,8 +173,8 @@ class Synchronizer {
 				if (this.itemIsStrictlyOlderThan(remote, local.updatedTime)) {
 					action.type = 'update';
 					action.dest = 'remote';
-					action.reason = sprintf('Remote (%s) was modified after last sync of local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString(),);
+					action.reason = sprintf('Remote (%s) was modified before updated time of local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString(),);
-				} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) {
+				} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && this.itemIsStrictlyNewerThan(local, local.syncTime)) {
 					action.type = 'conflict';
 					action.reason = sprintf('Both remote (%s) and local (%s) were modified after the last sync (%s).',
 						moment.unix(remote.updatedTime).toISOString(),
@@ -195,6 +193,10 @@ class Synchronizer {
 							{ type: 'update', dest: 'local' },
 						];
 					}
+				} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime) && local.updatedTime <= local.syncTime) {
+					action.type = 'update';
+					action.dest = 'local';
+					action.reason = sprintf('Remote (%s) was modified after update time of local (%s). And sync time (%s) is the same or more recent than local update time', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString());
 				} else {
 					continue; // Neither local nor remote item have been changed recently
 				}
@@ -230,7 +232,10 @@ class Synchronizer {
 				// modified since the last sync, it's been processed in the previous loop.
 				// So throw an exception is this normally impossible condition happens anyway.
 				// It's handled at condition this.itemIsStrictlyNewerThan(remote, local.syncTime) in above loop
-				if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) throw new Error('Remote cannot be newer than last sync time.');
+				if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) {
+					console.error('Remote cannot be newer than last sync time', remote, local);
+					throw new Error('Remote cannot be newer than last sync time');
+				}
 				
 				if (this.itemIsStrictlyNewerThan(remote, local.updatedTime)) {
 					action.type = 'update';
@@ -285,53 +290,49 @@ class Synchronizer {
 			if (action.type == 'create') {
 				if (action.dest == 'remote') {
 					let content = null;
+					let dbItem = syncItem.dbItem;
 
 					if (syncItem.type == 'folder') {
-						content = Folder.toFriendlyString(syncItem.dbItem);
+						content = Folder.toFriendlyString(dbItem);
 					} else {
-						content = Note.toFriendlyString(syncItem.dbItem);
+						content = Note.toFriendlyString(dbItem);
 					}
 
 					return this.api().put(path, content).then(() => {
-						return this.api().setTimestamp(path, syncItem.updatedTime);
+						return this.api().setTimestamp(path, dbItem.updated_time);
 					});
+
+					// TODO: save sync_time
 				} else {
 					let dbItem = syncItem.remoteItem.content;
 					dbItem.sync_time = time.unix();
-					dbItem.updated_time = dbItem.sync_time;
+					dbItem.updated_time = action.remote.updatedTime;
 					if (syncItem.type == 'folder') {
 						return Folder.save(dbItem, { isNew: true, autoTimestamp: false });
 					} else {
 						return Note.save(dbItem, { isNew: true, autoTimestamp: false });
 					}
+
+					// TODO: save sync_time
 				}
 			}
 
 			if (action.type == 'update') {
 				if (action.dest == 'remote') {
-					// let content = null;
+					let dbItem = syncItem.dbItem;
-
+					let ItemClass = BaseItem.itemClass(dbItem);
-					// if (syncItem.type == 'folder') {
+					let content = ItemClass.toFriendlyString(dbItem);
-					// 	content = Folder.toFriendlyString(syncItem.dbItem);
+					//console.info('PUT', content);
-					// } else {
+					return this.api().put(path, content).then(() => {
-					// 	content = Note.toFriendlyString(syncItem.dbItem);
+						return this.api().setTimestamp(path, dbItem.updated_time);
-					// }
+					}).then(() => {
-
+						let toSave = { id: dbItem.id, sync_time: time.unix() };
-					// return this.api().put(path, content).then(() => {
+						return NoteFolderService.save(syncItem.type, dbItem, null, { autoTimestamp: false });
-					// 	return this.api().setTimestamp(path, syncItem.updatedTime);
+					});
-					// });
 				} else {
-					let dbItem = syncItem.remoteItem.content;
+					let dbItem = Object.assign({}, syncItem.remoteItem.content);
 					dbItem.sync_time = time.unix();
-					dbItem.updated_time = dbItem.sync_time;
 					return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem, { autoTimestamp: false });
-					// let dbItem = syncItem.remoteItem.content;
-					// dbItem.sync_time = time.unix();
-					// if (syncItem.type == 'folder') {
-					// 	return Folder.save(dbItem, { isNew: true });
-					// } else {
-					// 	return Note.save(dbItem, { isNew: true });
-					// }
 				}
 			}
 		}
@@ -346,12 +347,9 @@ class Synchronizer {
 		let action = this.syncAction(localItem, remoteItem, []);
 		await this.processSyncAction(action);
 
-		dbItem.sync_time = time.unix();
+		let toSave = Object.assign({}, dbItem);
-		if (localItem.type == 'folder') {
+		toSave.sync_time = time.unix();
-			return Folder.save(dbItem);
+		return NoteFolderService.save(localItem.type, toSave, dbItem, { autoTimestamp: false });
-		} else {
-			return Note.save(dbItem);
-		}
 	}
 
 	async processRemoteItem(remoteItem) {