testing sync

ReactNativeClient/src/base-model.js

@@ -1,6 +1,7 @@
 import { Log } from 'src/log.js';
 import { Database } from 'src/database.js';
 import { uuid } from 'src/uuid.js';
+import { time } from 'src/time-utils.js';
 
 class BaseModel {
 
@@ -99,6 +100,7 @@ class BaseModel {
 		}
 		if (!('trackChanges' in options)) options.trackChanges = true;
 		if (!('isNew' in options)) options.isNew = 'auto';
+		if (!('autoTimestamp' in options)) options.autoTimestamp = true;
 		return options;
 	}
 
@@ -137,6 +139,7 @@ class BaseModel {
 	static diffObjects(oldModel, newModel) {
 		let output = {};
 		for (let n in newModel) {
+			if (n == 'type_') continue;
 			if (!newModel.hasOwnProperty(n)) continue;
 			if (!(n in oldModel) || newModel[n] !== oldModel[n]) {
 				output[n] = newModel[n];
@@ -145,9 +148,7 @@ class BaseModel {
 		return output;
 	}
 
-	static saveQuery(o, isNew = 'auto') {
-		if (isNew == 'auto') isNew = !o.id;
-
+	static saveQuery(o, options) {
 		let temp = {}
 		let fieldNames = this.fieldNames();
 		for (let i = 0; i < fieldNames.length; i++) {
@@ -156,22 +157,21 @@ class BaseModel {
 		}
 		o = temp;
 
-		let query = '';
+		let query = {};
 		let itemId = o.id;
 
-		if (!o.updated_time && this.hasField('updated_time')) {
-			o.updated_time = Math.round((new Date()).getTime() / 1000);
+		if (options.autoTimestamp && this.hasField('updated_time')) {
+			o.updated_time = time.unix();
 		}
 
-		if (isNew) {
+		if (options.isNew) {
 			if (this.useUuid() && !o.id) {
-				//o = Object.assign({}, o);
 				itemId = uuid.create();
 				o.id = itemId;
 			}
 
 			if (!o.created_time && this.hasField('created_time')) {
-				o.created_time = Math.round((new Date()).getTime() / 1000);
+				o.created_time = time.unix();
 			}
 
 			query = Database.insertQuery(this.tableName(), o);
@@ -192,10 +192,10 @@ class BaseModel {
 	static save(o, options = null) {
 		options = this.modOptions(options);
 
-		let isNew = options.isNew == 'auto' ? !o.id : options.isNew;
+		options.isNew = options.isNew == 'auto' ? !o.id : options.isNew;
 
 		let queries = [];
-		let saveQuery = this.saveQuery(o, isNew);
+		let saveQuery = this.saveQuery(o, options);
 		let itemId = saveQuery.id;
 
 		queries.push(saveQuery);

ReactNativeClient/src/database.js

@@ -253,6 +253,8 @@ class Database {
 	}
 
 	static insertQuery(tableName, data) {
+		if (!data || !Object.keys(data).length) throw new Error('Data is empty');
+
 		let keySql= '';
 		let valueSql = '';
 		let params = [];
@@ -271,6 +273,8 @@ class Database {
 	}
 
 	static updateQuery(tableName, data, where) {
+		if (!data || !Object.keys(data).length) throw new Error('Data is empty');
+
 		let sql = '';
 		let params = [];
 		for (let key in data) {

ReactNativeClient/src/file-api-driver-memory.js

@@ -1,3 +1,5 @@
+import { time } from 'src/time-utils.js';
+
 class FileApiDriverMemory {
 
 	constructor(baseDir) {
@@ -21,23 +23,23 @@ class FileApiDriverMemory {
 	}
 
 	newItem(path, isDir = false) {
+		let now = time.unix();
 		return {
 			path: path,
 			isDir: isDir,
-			updatedTime: this.currentTimestamp(),
-			createdTime: this.currentTimestamp(),
+			updatedTime: now,
+			createdTime: now,
 			content: '',
 		};
 	}
 
 	stat(path) {
-		let item = this.itemIndexByPath(path);
-		if (!item) return Promise.reject(new Error('File not found: ' + path));
-		return Promise.resolve(item);
+		let item = this.itemByPath(path);
+		return Promise.resolve(item ? Object.assign({}, item) : null);
 	}
 
 	setTimestamp(path, timestamp) {
-		let item = this.itemIndexByPath(path);
+		let item = this.itemByPath(path);
 		if (!item) return Promise.reject(new Error('File not found: ' + path));
 		item.updatedTime = timestamp;
 		return Promise.resolve();
@@ -53,7 +55,6 @@ class FileApiDriverMemory {
 				let s = item.path.substr(path.length + 1);
 				if (s.split('/').length === 1) {
 					let it = Object.assign({}, item);
-					it.path = it.path.substr(path.length + 1);
 					output.push(it);
 				}
 			}
@@ -64,7 +65,7 @@ class FileApiDriverMemory {
 
 	get(path) {
 		let item = this.itemByPath(path);
-		if (!item) return Promise.reject(new Error('File not found: ' + path));
+		if (!item) return Promise.resolve(null);
 		if (item.isDir) return Promise.reject(new Error(path + ' is a directory, not a file'));
 		return Promise.resolve(item.content);
 	}
@@ -84,6 +85,7 @@ class FileApiDriverMemory {
 			this.items_.push(item);
 		} else {
 			this.items_[index].content = content;
+			this.items_[index].updatedTime = time.unix();
 		}
 		return Promise.resolve();
 	}

ReactNativeClient/src/file-api.js

@@ -37,34 +37,38 @@ class FileApi {
 		});
 	}
 
-	list(path = '', recursive = false, context = null) {
-		let fullPath = this.fullPath_(path);
-		return this.driver_.list(fullPath).then((items) => {
-			items = this.scopeItemsToBaseDir_(items);
-			if (recursive) {
-				let chain = [];
-				for (let i = 0; i < items.length; i++) {
-					let item = items[i];
-					if (!item.isDir) continue;
-
-					chain.push(() => {
-						return this.list(item.path, true).then((children) => {
-							for (let j = 0; j < children.length; j++) {
-								let md = children[j];
-								md.path = item.path + '/' + md.path; 
-								items.push(md);
-							}
-						});
-					});
-				}
-
-				return promiseChain(chain).then(() => {
-					return items;
-				});
-			} else {
-				return items;
-			}
+	list() {
+		return this.driver_.list(this.baseDir_).then((items) => {
+			return this.scopeItemsToBaseDir_(items);
 		});
+		// let fullPath = this.fullPath_(path);
+		// return this.driver_.list(fullPath).then((items) => {
+		// 	return items;
+		// 	// items = this.scopeItemsToBaseDir_(items);
+		// 	// if (recursive) {
+		// 	// 	let chain = [];
+		// 	// 	for (let i = 0; i < items.length; i++) {
+		// 	// 		let item = items[i];
+		// 	// 		if (!item.isDir) continue;
+
+		// 	// 		chain.push(() => {
+		// 	// 			return this.list(item.path, true).then((children) => {
+		// 	// 				for (let j = 0; j < children.length; j++) {
+		// 	// 					let md = children[j];
+		// 	// 					md.path = item.path + '/' + md.path; 
+		// 	// 					items.push(md);
+		// 	// 				}
+		// 	// 			});
+		// 	// 		});
+		// 	// 	}
+
+		// 	// 	return promiseChain(chain).then(() => {
+		// 	// 		return items;
+		// 	// 	});
+		// 	// } else {
+		// 	// 	return items;
+		// 	// }
+		// });
 	}
 
 	setTimestamp(path, timestamp) {
@@ -77,7 +81,7 @@ class FileApi {
 	}
 
 	stat(path) {
-		console.info('stat ' + path);
+		//console.info('stat ' + path);
 		return this.driver_.stat(this.fullPath_(path)).then((output) => {
 			if (!output) return output;
 			output.path = path;
@@ -86,12 +90,12 @@ class FileApi {
 	}
 
 	get(path) {
-		console.info('get ' + path);
+		//console.info('get ' + path);
 		return this.driver_.get(this.fullPath_(path));
 	}
 
 	put(path, content) {
-		console.info('put ' + path);
+		//console.info('put ' + path);
 		return this.driver_.put(this.fullPath_(path), content);
 	}
 

ReactNativeClient/src/models/folder.js

@@ -74,9 +74,20 @@ class Folder extends BaseItem {
 		//return this.db().selectOne('SELECT * FROM notes WHERE `parent_id` = ? AND `' + field + '` = ?', [folderId, value]);
 	}
 
-	static all() {
-		return this.modelSelectAll('SELECT * FROM folders');
-		// return this.db().selectAll('SELECT * FROM folders');
+	static async all(includeNotes = false) {
+		let folders = await this.modelSelectAll('SELECT * FROM folders');
+		if (!includeNotes) return folders;
+
+		let output = [];
+		for (let i = 0; i < folders.length; i++) {
+			let folder = folders[i];
+			let notes = await Note.all(folder.id);
+			output.push(folder);
+			output = output.concat(notes);
+		}
+
+		return output;
+
 	}
 
 	static save(o, options = null) {

ReactNativeClient/src/models/note.js

@@ -69,6 +69,10 @@ class Note extends BaseItem {
 		});
 	}
 
+	static all(parentId) {
+		return this.modelSelectAll('SELECT * FROM notes WHERE parent_id = ?', [parentId]);
+	}
+
 	static save(o, options = null) {
 		return super.save(o, options).then((result) => {
 			// 'result' could be a partial one at this point (if, for example, only one property of it was saved)

ReactNativeClient/src/services/note-folder-service.js

@@ -11,7 +11,7 @@ import { Registry } from 'src/registry.js';
 
 class NoteFolderService extends BaseService {
 
-	static save(type, item, oldItem) {
+	static save(type, item, oldItem, options = null) {
 		let diff = null;
 		if (oldItem) {
 			diff = BaseModel.diffObjects(oldItem, item);
@@ -32,7 +32,9 @@ class NoteFolderService extends BaseService {
 			toSave.id = item.id;
 		}
 
-		return ItemClass.save(toSave).then((savedItem) => {
+		console.info(toSave);
+
+		return ItemClass.save(toSave, options).then((savedItem) => {
 			output = Object.assign(item, savedItem);
 			if (isNew && type == 'note') return Note.updateGeolocation(output.id);
 		}).then(() => {

ReactNativeClient/src/synchronizer.js

@@ -161,22 +161,21 @@ class Synchronizer {
 
 			if (!remote) {
 				if (local.syncTime) {
-					// The item has been synced previously and now is no longer in the dest
-					// which means it has been deleted.
 					action.type = 'delete';
 					action.dest = 'local';
+					action.reason = 'Local item has been synced to remote previously, but remote no longer exist, which means it has been deleted';
 				} else {
-					// The item has never been synced and is not present in the dest
-					// which means it is new
 					action.type = 'create';
 					action.dest = 'remote';
+					action.reason = 'Local item has never been synced to remote, and remote does not exists, which means it is new';
 				}
 			} else {
 				if (this.itemIsStrictlyOlderThan(local, local.syncTime)) continue;
 
-				if (this.itemIsStrictlyOlderThan(remote, local.syncTime)) {
+				if (this.itemIsStrictlyOlderThan(remote, local.updatedTime)) {
 					action.type = 'update';
 					action.dest = 'remote';
+					action.reason = sprintf('Remote (%s) was modified after last sync of local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString(),);
 				} else if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) {
 					action.type = 'conflict';
 					action.reason = sprintf('Both remote (%s) and local items (%s) were modified after the last sync (%s).',
@@ -186,10 +185,6 @@ class Synchronizer {
 					);
 
 					if (local.type == 'folder') {
-						// For folders, currently we don't completely handle conflicts, we just
-						// we just update the local dir (.folder metadata file) with the remote
-						// version. It means the local version is lost but shouldn't be a big deal
-						// and should be rare (at worst, the folder name needs to renamed).
 						action.solution = [
 							{ type: 'update', dest: 'local' },
 						];
@@ -230,10 +225,20 @@ class Synchronizer {
 				}
 			} else {
 				if (this.itemIsStrictlyOlderThan(remote, local.syncTime)) continue; // Already have this version
+
 				// Note: no conflict is possible here since if the local item has been
 				// modified since the last sync, it's been processed in the previous loop.
-				action.type = 'update';
-				action.dest = 'local';
+				// So throw an exception is this normally impossible condition happens anyway.
+				// It's handled at condition this.itemIsStrictlyNewerThan(remote, local.syncTime) in above loop
+				if (this.itemIsStrictlyNewerThan(remote, local.syncTime)) throw new Error('Remote item cannot be newer than last sync time.');
+
+				if (this.itemIsStrictlyNewerThan(remote, local.updatedTime)) {
+					action.type = 'update';
+					action.dest = 'local';
+					action.reason = sprintf('Remote (%s) was modified after last sync of local (%s).', moment.unix(remote.updatedTime).toISOString(), moment.unix(local.syncTime).toISOString(),);;
+				} else {
+					continue;
+				}
 			}
 
 			output.push(action);
@@ -268,7 +273,7 @@ class Synchronizer {
 
 		if (!action) return Promise.resolve();
 
-		console.info('Sync action: ' + action.type + ' ' + action.dest);
+		console.info('Sync action: ' + action.type + ' ' + action.dest + ': ' + action.reason);
 
 		if (action.type == 'conflict') {
 			console.info(action);
@@ -293,10 +298,11 @@ class Synchronizer {
 				} else {
 					let dbItem = syncItem.remoteItem.content;
 					dbItem.sync_time = time.unix();
+					dbItem.updated_time = dbItem.sync_time;
 					if (syncItem.type == 'folder') {
-						return Folder.save(dbItem, { isNew: true });
+						return Folder.save(dbItem, { isNew: true, autoTimestamp: false });
 					} else {
-						return Note.save(dbItem, { isNew: true });
+						return Note.save(dbItem, { isNew: true, autoTimestamp: false });
 					}
 				}
 			}
@@ -317,7 +323,8 @@ class Synchronizer {
 				} else {
 					let dbItem = syncItem.remoteItem.content;
 					dbItem.sync_time = time.unix();
-					return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem);
+					dbItem.updated_time = dbItem.sync_time;
+					return NoteFolderService.save(syncItem.type, dbItem, action.local.dbItem, { autoTimestamp: false });
 					// let dbItem = syncItem.remoteItem.content;
 					// dbItem.sync_time = time.unix();
 					// if (syncItem.type == 'folder') {
@@ -349,6 +356,7 @@ class Synchronizer {
 
 	async processRemoteItem(remoteItem) {
 		let content = await this.api().get(remoteItem.path);
+		if (!content) throw new Error('Cannot get content for: ' + remoteItem.path);
 		remoteItem.content = Note.fromFriendlyString(content);
 		let remoteSyncItem = this.remoteItemToSyncItem(remoteItem);
 
@@ -362,6 +370,7 @@ class Synchronizer {
 	async processState_uploadChanges() {
 		while (true) {
 			let result = await NoteFolderService.itemsThatNeedSync(50);
+			console.info('Items that need sync: ' + result.items.length);
 			for (let i = 0; i < result.items.length; i++) {
 				let item = result.items[i];
 				await this.processLocalItem(item);
@@ -370,6 +379,8 @@ class Synchronizer {
 			if (!result.hasMore) break;
 		}
 
+		//console.info('DOWNLOAD CHANGE DISABLED'); return Promise.resolve(); 
+		
 		return this.processState('downloadChanges');
 	}
 
@@ -396,7 +407,10 @@ class Synchronizer {
 		// 	return;
 		// }
 
-		return this.processState('uploadChanges');
+		return this.processState('uploadChanges').catch((error) => {
+			console.info('Synchronizer error:', error);
+			throw error;
+		});
 	}