Test sync

CliClient/.gitignore

@@ -1,4 +1,5 @@
 build/
 node_modules/
 app/src
-spec-build/
+tests-build/
+tests/src

CliClient/package.json

@@ -33,6 +33,6 @@
     "babelbuild": "babel app -d build",
     "build": "babel-changed app -d build --source-maps && babel-changed app/src/models -d build/src/models --source-maps && babel-changed app/src/services -d build/src/services --source-maps",
     "clean": "babel-changed --reset",
-    "test": "babel-changed spec -d spec-build --source-maps && jasmine"
+    "test": "babel-changed tests -d tests-build --source-maps && jasmine"
   }
 }

CliClient/run_test.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-rm -f "$CLIENT_DIR/spec-build/src"
-ln -s "$CLIENT_DIR/build/src" "$CLIENT_DIR/spec-build"
+rm -f "$CLIENT_DIR/tests-build/src"
+mkdir -p "$CLIENT_DIR/tests-build"
+ln -s "$CLIENT_DIR/build/src" "$CLIENT_DIR/tests-build"
 
-npm build && NODE_PATH="$CLIENT_DIR/spec-build/" npm test spec-build/synchronizer.js
+npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js

CliClient/spec/src

@@ -1 +0,0 @@
-/home/laurent/src/notes/CliClient/../ReactNativeClient/src

CliClient/spec/synchronizer.js

@@ -1,32 +0,0 @@
-import { Synchronizer } from 'src/synchronizer.js';
-import { FileApi } from 'src/file-api.js';
-import { FileApiDriverMemory } from 'src/file-api-driver-memory.js';
-
-describe("syncActions", function() {
-
-	let fileDriver = new FileApiDriverMemory();
-	let fileApi = new FileApi('/root', fileDriver);
-	let synchronizer = new Synchronizer(null, fileApi);
-
-	it("and so is a spec", function() {
-		let localItems = [];
-		localItems.push({ path: 'test', isDir: true, updatedTime: 1497370000 });
-		localItems.push({ path: 'test/un', updatedTime: 1497370000 });
-		localItems.push({ path: 'test/deux', updatedTime: 1497370000 });
-
-		let remoteItems = [];
-
-		let actions = synchronizer.syncActions(localItems, remoteItems, 0);
-
-		expect(actions.length).toBe(3);
-		
-
-
-
-		// synchronizer.format();
-		// synchronizer.mkdir('test');
-		// synchronizer.touch('test/un');
-		// synchronizer.touch('test/deux');
-		// synchronizer.touch('test/trois');
-	});
-});

CliClient/tests/synchronizer.js

@@ -0,0 +1,160 @@
+import { Synchronizer } from 'src/synchronizer.js';
+import { FileApi } from 'src/file-api.js';
+import { FileApiDriverMemory } from 'src/file-api-driver-memory.js';
+import { time } from 'src/time-utils.js';
+
+describe('Synchronizer syncActions', function() {
+
+	let fileDriver = new FileApiDriverMemory();
+	let fileApi = new FileApi('/root', fileDriver);
+	let synchronizer = new Synchronizer(null, fileApi);
+
+	// Note: set 1 matches set 1 of createRemoteItems()
+	function createLocalItems(id, updatedTime, lastSyncTime) {
+		let output = [];
+		if (id === 1) {
+			output.push({ path: 'test', isDir: true, updatedTime: updatedTime, lastSyncTime: lastSyncTime });
+			output.push({ path: 'test/un', updatedTime: updatedTime, lastSyncTime: lastSyncTime });
+		} else {
+			throw new Error('Invalid ID');
+		}
+		return output;
+	}
+
+	function createRemoteItems(id = 1, updatedTime = null) {
+		if (!updatedTime) updatedTime = time.unix();
+
+		if (id === 1) {
+			return fileApi.format()
+			.then(() => fileApi.mkdir('test'))
+			.then(() => fileApi.put('test/un', 'abcd'))
+			.then(() => fileApi.list('', true))
+			.then((items) => {
+				for (let i = 0; i < items.length; i++) {
+					items[i].updatedTime = updatedTime;
+				}
+				return items;
+			});
+		} else {
+			throw new Error('Invalid ID');
+		}
+	}
+
+	it('should create remote items', function() {
+		let localItems = createLocalItems(1, time.unix(), 0);
+		let remoteItems = [];
+
+		let actions = synchronizer.syncActions(localItems, remoteItems, []);
+
+		expect(actions.length).toBe(2);
+		for (let i = 0; i < actions.length; i++) {
+			expect(actions[i].type).toBe('create');
+			expect(actions[i].dest).toBe('remote');
+		}
+	});
+
+	it('should update remote items', function(done) {	
+		createRemoteItems(1).then((remoteItems) => {
+			let lastSyncTime = time.unix() + 1000;
+			let localItems = createLocalItems(1, lastSyncTime + 1000, lastSyncTime);
+			let actions = synchronizer.syncActions(localItems, remoteItems, []);
+
+			expect(actions.length).toBe(2);
+			for (let i = 0; i < actions.length; i++) {
+				expect(actions[i].type).toBe('update');
+				expect(actions[i].dest).toBe('remote');
+			}
+
+			done();
+		});
+	});
+
+	it('should detect conflict', function(done) {
+		// Simulate this scenario:
+		// - Client 1 create items
+		// - Client 1 sync
+		// - Client 2 sync
+		// - Client 2 change items
+		// - Client 2 sync
+		// - Client 1 change items
+		// - Client 1 sync
+		// => Conflict
+
+		createRemoteItems(1).then((remoteItems) => {
+			let localItems = createLocalItems(1, time.unix() + 1000, time.unix() - 1000);
+			let actions = synchronizer.syncActions(localItems, remoteItems, []);
+
+			expect(actions.length).toBe(2);
+			for (let i = 0; i < actions.length; i++) {
+				expect(actions[i].type).toBe('conflict');
+			}
+
+			done();
+		});
+	});
+
+
+	it('should create local file', function(done) {
+		createRemoteItems(1).then((remoteItems) => {
+			let localItems = [];
+			let actions = synchronizer.syncActions(localItems, remoteItems, []);
+
+			expect(actions.length).toBe(2);
+			for (let i = 0; i < actions.length; i++) {
+				expect(actions[i].type).toBe('create');
+				expect(actions[i].dest).toBe('local');
+			}
+
+			done();
+		});
+	});
+
+	it('should delete remote files', function(done) {
+		createRemoteItems(1).then((remoteItems) => {
+			let localItems = createLocalItems(1, time.unix(), time.unix());
+			let deletedItemPaths = [localItems[0].path, localItems[1].path];
+			let actions = synchronizer.syncActions([], remoteItems, deletedItemPaths);
+
+			expect(actions.length).toBe(2);
+			for (let i = 0; i < actions.length; i++) {
+				expect(actions[i].type).toBe('delete');
+				expect(actions[i].dest).toBe('remote');
+			}
+
+			done();
+		});
+	});
+
+	it('should delete local files', function(done) {
+		let lastSyncTime = time.unix();
+		createRemoteItems(1, lastSyncTime - 1000).then((remoteItems) => {
+			let localItems = createLocalItems(1, lastSyncTime - 1000, lastSyncTime);
+			let actions = synchronizer.syncActions(localItems, [], []);
+
+			expect(actions.length).toBe(2);
+			for (let i = 0; i < actions.length; i++) {
+				expect(actions[i].type).toBe('delete');
+				expect(actions[i].dest).toBe('local');
+			}
+
+			done();
+		});
+	});
+
+	it('should update local files', function(done) {
+		let lastSyncTime = time.unix();
+		createRemoteItems(1, lastSyncTime + 1000).then((remoteItems) => {
+			let localItems = createLocalItems(1, lastSyncTime - 1000, lastSyncTime);
+			let actions = synchronizer.syncActions(localItems, remoteItems, []);
+
+			expect(actions.length).toBe(2);
+			for (let i = 0; i < actions.length; i++) {
+				expect(actions[i].type).toBe('update');
+				expect(actions[i].dest).toBe('local');
+			}
+
+			done();
+		});
+	});
+
+});

ReactNativeClient/src/synchronizer.js

@@ -96,35 +96,103 @@ class Synchronizer {
 		};
 	}
 
+	itemIsSameDate(item, date) {
+		return Math.abs(item.updatedTime - date) <= 1;
+	}
+
 	itemIsNewerThan(item, date) {
+		if (this.itemIsSameDate(item, date)) return false;
 		return item.updatedTime > date;
 	}
 
 	itemIsOlderThan(item, date) {
-		return !this.itemIsNewerThan(item, date);
+		if (this.itemIsSameDate(item, date)) return false;
+		return item.updatedTime < date;
 	}
 
-	syncActions(localItems, remoteItems, lastSyncTime) {
+	// Assumption: it's not possible to, for example, have a directory one the dest
+	// and a file with the same name on the source. It's not possible because the
+	// file and directory names are UUID so should be unique.
+	syncActions(localItems, remoteItems, deletedLocalPaths) {
 		let output = [];
+		let donePaths = [];
 
 		for (let i = 0; i < localItems.length; i++) {
-			let item = localItems[i];
-			let remoteItem = this.itemByPath(remoteItems, item.path);
+			let local = localItems[i];
+			let remote = this.itemByPath(remoteItems, local.path);
+
 			let action = {
-				localItem: item,
-				remoteItem: remoteItem,
+				local: local,
+				remote: remote,
 			};
-			if (!remoteItem) {
+
+			if (!remote) {
+				if (local.lastSyncTime) {
+					// The item has been synced previously and now is no longer in the dest
+					// which means it has been deleted.
+					action.type = 'delete';
+					action.dest = 'local';
+				} else {
+					// The item has never been synced and is not present in the dest
+					// which means it is new
 					action.type = 'create';
-				action.where = 'there';
-			} else {
-				if (this.itemIsOlderThan(remoteItem, lastSyncTime)) {
-					action.type = 'update';
-					action.where = 'there';
-				} else {
-					action.type = 'conflict'; // Move local to /Conflict; Copy remote here
-					action.where = 'here';
+					action.dest = 'remote';
 				}
+			} else {
+				if (this.itemIsOlderThan(local, local.lastSyncTime)) continue;
+
+				if (this.itemIsOlderThan(remote, local.lastSyncTime)) {
+					action.type = 'update';
+					action.dest = 'remote';
+				} else {
+					action.type = 'conflict';
+					if (local.isDir) {
+						// For folders, currently we don't completely handle conflicts, we just
+						// we just update the local dir (.folder metadata file) with the remote
+						// version. It means the local version is lost but shouldn't be a big deal
+						// and should be rare (at worst, the folder name needs to renamed).
+						action.solution = [
+							{ type: 'update', dest: 'local' },
+						];
+					} else {
+						action.solution = [
+							{ type: 'copy-to-remote-conflict-dir', dest: 'local' },
+							{ type: 'copy-to-local-conflict-dir', dest: 'local' },
+							{ type: 'update', dest: 'local' },
+						];
+					}
+				}
+			}
+
+			donePaths.push(local.path);
+
+			output.push(action);
+		}
+
+		for (let i = 0; i < remoteItems.length; i++) {
+			let remote = remoteItems[i];
+			if (donePaths.indexOf(remote.path) >= 0) continue; // Already handled in the previous loop
+			let local = this.itemByPath(localItems, remote.path);
+
+			let action = {
+				local: local,
+				remote: remote,
+			};
+
+			if (!local) {
+				if (deletedLocalPaths.indexOf(remote.path) >= 0) {
+					action.type = 'delete';
+					action.dest = 'remote';
+				} else {
+					action.type = 'create';
+					action.dest = 'local';
+				}
+			} else {
+				if (this.itemIsOlderThan(remote, local.lastSyncTime)) continue; // Already have this version
+				// Note: no conflict is possible here since if the local item has been
+				// modified since the last sync, it's been processed in the previous loop.
+				action.type = 'update';
+				action.dest = 'local';
 			}
 
 			output.push(action);

ReactNativeClient/src/time-utils.js

@@ -0,0 +1,9 @@
+let time = {
+
+	unix() {
+		return Math.round((new Date()).getTime() / 1000);
+	}
+
+}
+
+export { time };

joplin.sublime-project

@@ -11,8 +11,9 @@
 				"app/data/uploads",
 				"CliClient/node_modules",
 				"CliClient/build",
-				"CliClient/spec-build",
+				"CliClient/tests-build",
 				"CliClient/app/src",
+				"CliClient/tests/src",
 				"ReactNativeClient/node_modules",
 				"ReactNativeClient/android/app/build",
 				"ReactNativeClient/android/build",