Synchronizer

ReactNativeClient/src/base-model.js

@@ -58,13 +58,17 @@ class BaseModel {
 		return options;
 	}
 
+	static load(id) {
+		return this.db().selectOne('SELECT * FROM ' + this.tableName() + ' WHERE id = ?', [id]);
+	}
+
 	static saveQuery(o, isNew = 'auto') {
 		if (isNew == 'auto') isNew = !o.id;
 		let query = '';
 		let itemId = o.id;
 
 		if (isNew) {
-			if (this.useUuid()) {
+			if (this.useUuid() && !o.id) {
 				o = Object.assign({}, o);
 				itemId = uuid.create();
 				o.id = itemId;
@@ -96,15 +100,28 @@ class BaseModel {
 				// which are not handled by React Native.
 				const { Change } = require('src/models/change.js');
 
+				if (isNew) {
 					let change = Change.newChange();
-				change.type = isNew ? Change.TYPE_CREATE : Change.TYPE_UPDATE;
+					change.type = Change.TYPE_CREATE;
 					change.item_id = query.id;
 					change.item_type = this.itemType();
 
 					let changeQuery = Change.saveQuery(change);
 					tx.executeSql(changeQuery.sql, changeQuery.params);
+				} else {
+					for (let n in o) {
+						if (!o.hasOwnProperty(n)) continue;
 
-				// TODO: item field for UPDATE
+						let change = Change.newChange();
+						change.type = Change.TYPE_UPDATE;
+						change.item_id = query.id;
+						change.item_type = this.itemType();
+						change.item_field = n;
+
+						let changeQuery = Change.saveQuery(change);
+						tx.executeSql(changeQuery.sql, changeQuery.params);
+					}
+				}
 			}
 		}).then(() => {
 			o = Object.assign({}, o);
@@ -122,16 +139,16 @@ class BaseModel {
 		}
 
 		return this.db().exec('DELETE FROM ' + this.tableName() + ' WHERE id = ?', [id]).then(() => {
-			// if (options.trackChanges && this.trackChanges()) {
+			if (options.trackChanges && this.trackChanges()) {
-			// 	const { Change } = require('src/models/change.js');
+				const { Change } = require('src/models/change.js');
 
-			// 	let change = Change.newChange();
+				let change = Change.newChange();
-			// 	change.type = Change.TYPE_DELETE;
+				change.type = Change.TYPE_DELETE;
-			// 	change.item_id = id;
+				change.item_id = id;
-			// 	change.item_type = this.itemType();
+				change.item_type = this.itemType();
 
-			// 	return Change.save(change);
+				return Change.save(change);
-			// }
+			}
 		});
 	}
 

ReactNativeClient/src/components/item-list.js

@@ -72,7 +72,7 @@ class ItemListComponent extends Component {
 				<TouchableHighlight onPress={onPress} onLongPress={onLongPress}>
 					<View>
 						{ isEditable  && <Checkbox label={item.title} ></Checkbox> }
-						{ !isEditable && <Text>{item.title}</Text> }
+						{ !isEditable && <Text>{item.title} [{item.id}]</Text> }
 					</View>
 				</TouchableHighlight>
 			);

ReactNativeClient/src/database.js

@@ -110,7 +110,7 @@ class Database {
 	}
 
 	open() {
-		this.db_ = SQLite.openDatabase({ name: '/storage/emulated/0/Download/joplin-11.sqlite' }, (db) => {
+		this.db_ = SQLite.openDatabase({ name: '/storage/emulated/0/Download/joplin-12.sqlite' }, (db) => {
 			Log.info('Database was open successfully');
 		}, (error) => {
 			Log.error('Cannot open database: ', error);

ReactNativeClient/src/main.js

@@ -15,6 +15,7 @@ function main() {
 	Registry.setDebugMode(true);
 	AppRegistry.registerComponent('AwesomeProject', () => Root);
 	Log.setLevel(Registry.debugMode() ? Log.LEVEL_DEBUG : Log.LEVEL_WARN);
+	Log.info('START ======================================================================================================');
 	// Note: The final part of the initialization process is in
 	// AppComponent.componentDidMount(), when the application is ready.
 }

ReactNativeClient/src/models/change.js

@@ -3,7 +3,7 @@ import { Log } from 'src/log.js';
 
 class Change extends BaseModel {
 
-	static TYPE_UNKNOWN = 0;
+	static TYPE_NOOP = 0;
 	static TYPE_CREATE = 1;
 	static TYPE_UPDATE = 2;
 	static TYPE_DELETE = 3;
@@ -22,15 +22,84 @@ class Change extends BaseModel {
 		};
 	}
 
-	// static all() {
+	static all() {
-	// 	return this.db().selectAll('SELECT * FROM folders').then((r) => {
+		return this.db().selectAll('SELECT * FROM changes').then((r) => {
-	// 		let output = [];
+			let output = [];
-	// 		for (let i = 0; i < r.rows.length; i++) {
+			for (let i = 0; i < r.rows.length; i++) {
-	// 			output.push(r.rows.item(i));
+				output.push(r.rows.item(i));
-	// 		}
+			}
-	// 		return output;
+			return output;
-	// 	});
+		});
-	// }
+	}
+
+	static deleteMultiple(ids) {
+		if (ids.length == 0) return Promise.resolve();
+
+		return this.db().transaction((tx) => {
+			let sql = '';
+			for (let i = 0; i < ids.length; i++) {
+				tx.executeSql('DELETE FROM changes WHERE id = ?', [ids[i]]);
+			}			
+		});
+	}
+
+	static mergeChanges(changes) {
+		let createdItems = [];
+		let deletedItems = [];
+		let itemChanges = {};
+
+		for (let i = 0; i < changes.length; i++) {
+			let change = changes[i];
+
+			if (itemChanges[change.item_id]) {
+				mergedChange = itemChanges[change.item_id];
+			} else {
+				mergedChange = {
+					item_id: change.item_id,
+					item_type: change.item_type,
+					fields: [],
+					ids: [],
+					type: change.type,
+				}
+			}
+
+			if (change.type == this.TYPE_CREATE) {
+				createdItems.push(change.item_id);
+			} else if (change.type == this.TYPE_DELETE) {
+				deletedItems.push(change.item_id);
+			} else if (change.type == this.TYPE_UPDATE) {
+				if (mergedChange.fields.indexOf(change.item_field) < 0) {
+					mergedChange.fields.push(change.item_field);
+				}
+			}
+
+			mergedChange.ids.push(change.id);
+
+			itemChanges[change.item_id] = mergedChange;
+		}
+
+		let output = [];
+
+		for (let itemId in itemChanges) {
+			if (!itemChanges.hasOwnProperty(itemId)) continue;
+			let change = itemChanges[itemId];
+
+			if (createdItems.indexOf(itemId) >= 0 && deletedItems.indexOf(itemId) >= 0) {
+				// Item both created then deleted - skip
+				change.type = this.TYPE_NOOP;
+			} else if (deletedItems.indexOf(itemId) >= 0) {
+				// Item was deleted at some point - just return one 'delete' event
+				change.type = this.TYPE_DELETE;
+			} else if (createdItems.indexOf(itemId) >= 0) {
+				// Item was created then updated - just return one 'create' event with the latest changes
+				change.type = this.TYPE_CREATE;
+			}
+
+			output.push(change);
+		}
+
+		return output;
+	}
 
 }
 

ReactNativeClient/src/models/setting.js

@@ -43,15 +43,11 @@ class Setting extends BaseModel {
 	}
 
 	static setValue(key, value) {
-		// if (value !== null && typeof value === 'object') {
-		// 	return this.setObject(key, value);
-		// }
-
-		this.scheduleUpdate();
-
 		for (let i = 0; i < this.cache_.length; i++) {
 			if (this.cache_[i].key == key) {
+				if (this.cache_[i].value === value) return;
 				this.cache_[i].value = value;
+				this.scheduleUpdate();
 				return;
 			}
 		}
@@ -59,19 +55,9 @@ class Setting extends BaseModel {
 		let s = this.defaultSetting(key);
 		s.value = value;
 		this.cache_.push(s);
+		this.scheduleUpdate();
 	}
 
-	// static del(key) {
-	// 	this.scheduleUpdate();
-
-	// 	for (let i = 0; i < this.cache_.length; i++) {
-	// 		if (this.cache_[i].key == key) {
-	// 			this.cache_[i].value = value;
-	// 			return;
-	// 		}
-	// 	}
-	// }
-
 	static value(key) {
 		for (let i = 0; i < this.cache_.length; i++) {
 			if (this.cache_[i].key == key) {
@@ -104,13 +90,14 @@ class Setting extends BaseModel {
 		}
 	}
 
-	static scheduleUpdate() {
+	static saveAll() {
-		if (this.updateTimeoutId) clearTimeout(this.updateTimeoutId);
+		if (!this.updateTimeoutId_) return Promise.resolve();
 
-		this.updateTimeoutId = setTimeout(() => {
 		Log.info('Saving settings...');
-			this.updateTimeoutId = null;
+		clearTimeout(this.updateTimeoutId_);
-			BaseModel.db().transaction((tx) => {
+		this.updateTimeoutId_ = null;
+
+		return BaseModel.db().transaction((tx) => {
 			tx.executeSql('DELETE FROM settings');
 			for (let i = 0; i < this.cache_.length; i++) {
 				let q = Database.insertQuery(this.tableName(), this.cache_[i]);
@@ -119,8 +106,16 @@ class Setting extends BaseModel {
 		}).then(() => {
 			Log.info('Settings have been saved.');
 		}).catch((error) => {
-				Log.warn('Could not update settings:', error);
+			Log.warn('Could not save settings', error);
+			reject(error);
 		});
+	}
+
+	static scheduleUpdate() {
+		if (this.updateTimeoutId_) clearTimeout(this.updateTimeoutId_);
+
+		this.updateTimeoutId_ = setTimeout(() => {
+			this.saveAll();
 		}, 500);
 	}
 

ReactNativeClient/src/root.js

@@ -32,7 +32,7 @@ let defaultState = {
 };
 
 const reducer = (state = defaultState, action) => {
-	Log.info('Reducer action', action);
+	Log.info('Reducer action', action.type);
 
 	let newState = state;
 
@@ -163,8 +163,8 @@ class AppComponent extends React.Component {
 
 	componentDidMount() {
 		let db = new Database();
-		db.setDebugEnabled(Registry.debugMode());
+		//db.setDebugEnabled(Registry.debugMode());
-
+		db.setDebugEnabled(false);
 		BaseModel.dispatch = this.props.dispatch;
 
 		db.open().then(() => {
@@ -200,8 +200,8 @@ class AppComponent extends React.Component {
 				Log.warn('Cannot load folders', error);
 			});
 		}).then(() => {
-			// let synchronizer = new Synchronizer();
+			let synchronizer = new Synchronizer(db, Registry.api());
-			// synchronizer.start();
+			synchronizer.start();
 		}).catch((error) => {
 			Log.error('Initialization error:', error);
 		});

ReactNativeClient/src/synchronizer.js

@@ -3,11 +3,14 @@ import { Log } from 'src/log.js';
 import { Setting } from 'src/models/setting.js';
 import { Change } from 'src/models/change.js';
 import { Folder } from 'src/models/folder.js';
+import { promiseChain } from 'src/promise-chain.js';
 
 class Synchronizer {
 
-	constructor() {
+	constructor(db, api) {
 		this.state_ = 'idle';
+		this.db_ = db;
+		this.api_ = api;
 	}
 
 	state() {
@@ -15,52 +18,105 @@ class Synchronizer {
 	}
 
 	db() {
-		return Registry.db();
+		return this.db_;
 	}
 
 	api() {
-		return Registry.api();
+		return this.api_;
 	}
 
 	switchState(state) {
 		Log.info('Sync: switching state to: ' + state);
 
 		if (state == 'downloadChanges') {
+			let maxRevId = null;
 			this.api().get('synchronizer', { last_id: Setting.value('sync.lastRevId') }).then((syncOperations) => {
-				let promise = new Promise((resolve, reject) => { resolve(); });
+				let chain = [];
 				for (let i = 0; i < syncOperations.items.length; i++) {
 					let syncOp = syncOperations.items[i];
+					if (syncOp.id > maxRevId) maxRevId = syncOp.id;
 					if (syncOp.item_type == 'folder') {
+
 						if (syncOp.type == 'create') {
-							promise = promise.then(() => {
+							chain.push(() => {
 								let folder = Folder.fromApiResult(syncOp.item);
 								// TODO: automatically handle NULL fields by checking type and default value of field
 								if (!folder.parent_id) folder.parent_id = '';
 								return Folder.save(folder, { isNew: true });
 							});
 						}
-					}
-				}
 
-				promise.then(() => {
+						// TODO: update
+						// TODO: delete
+					}
+				}
+				return promiseChain(chain);
+			}).then(() => {
 				Log.info('All items synced.');
+				if (maxRevId) {
+					Setting.setValue('sync.lastRevId', maxRevId);
+					return Setting.saveAll();
+				}
+			}).then(() => {
+				this.switchState('uploadingChanges');
 			}).catch((error) => {
 				Log.warn('Sync error', error);
 			});
-			});
+		} else if (state == 'uploadingChanges') {
-		} else {
+			Change.all().then((changes) => {
+				let mergedChanges = Change.mergeChanges(changes);
+				// Log.info(mergedChanges);
+				let chain = [];
+				let processedChangeIds = [];
+				for (let i = 0; i < mergedChanges.length; i++) {
+					let c = mergedChanges[i];
+					chain.push(() => {
+						let p = null;
 
+						Log.info(this.api());
+
+						if (c.type == Change.TYPE_NOOP) {
+							p = Promise.resolve();
+						} else if (c.type == Change.TYPE_CREATE) {
+							p = Folder.load(c.item_id).then((folder) => {
+								return this.api().put('folders/' + folder.id, null, folder);
+							});
+						} else if (c.type == Change.TYPE_UPDATE) {
+							p = Folder.load(c.item_id).then((folder) => {
+								return this.api().patch('folders/' + folder.id, null, folder);
+							});
+						} else if (c.type == Change.TYPE_DELETE) {
+							p = Folder.load(c.item_id).then((folder) => {
+								return this.api().delete('folders/' + folder.id);
+							});
+						}
+
+						return p.then(() => {
+							processedChangeIds = processedChangeIds.concat(c.ids);
+						});
+					});
+				}
+
+				promiseChain(chain).then(() => {
+					Log.info('IDs to delete: ', processedChangeIds);
+					Change.deleteMultiple(processedChangeIds);
+				});
+			});
 		}
 	}
 
 	start() {
+		Log.info('Sync: start');
 
 		if (this.state() != 'idle') {
 			Log.info("Sync: cannot start synchronizer because synchronization already in progress. State: " + this.state());
 			return;
 		}
 
-		Log.info('Sync: start');
+		if (!this.api().session()) {
+			Log.info("Sync: cannot start synchronizer because user is not logged in.");
+			return;
+		}
 
 		this.switchState('downloadChanges');
 	}

ReactNativeClient/src/web-api.js

@@ -23,11 +23,18 @@ class WebApi {
 		let options = {};
 		options.method = method.toUpperCase();
 		if (data) {
-			var formData = new FormData();
+			let formData = null;
+			if (method == 'POST') {
+				formData = new FormData();
 				for (var key in data) {
 					if (!data.hasOwnProperty(key)) continue;
 					formData.append(key, data[key]);
 				}
+			} else {
+				options.headers = { 'Content-Type': 'application/x-www-form-urlencoded' };
+				formData = stringify(data);
+			}
+
 			options.body = formData;
 		}
 
@@ -91,6 +98,14 @@ class WebApi {
 		return this.exec('POST', path, query, data);
 	}
 
+	put(path, query, data) {
+		return this.exec('PUT', path, query, data);
+	}
+
+	patch(path, query, data) {
+		return this.exec('PATCH', path, query, data);
+	}
+
 	delete(path, query) {
 		return this.exec('DELETE', path, query);
 	}

src/AppBundle/Controller/ApiController.php

@@ -32,24 +32,31 @@ abstract class ApiController extends Controller {
 				$r->send();
 				echo "\n";
 			} else {
+				$msg = $e->getMessage();
+
+				// If the message was sent in Latin encoding, JsonResponse below will fail
+				// so encode it using UTF-8 here.
+				if (json_encode($msg) === false) {
+					$msg = utf8_encode($e->getMessage());
+				}
+
 				$r = array(
-					'error' => $e->getMessage(),
+					'error' => $msg,
 					'code' => 0,
 					'type' => 'Exception',
 					//'trace' => $e->getTraceAsString(),
 				);
+
+				try {
 					$response = new JsonResponse($r);
+				} catch (\Exception $wat) {
+					// If that happens, print the error message as is, since it's better than showing nothing at all
+					die($e->getMessage());
+				}
+
 				$response->setStatusCode(500);
 				$response->send();
 				echo "\n";
-
-
-				// $msg = array();
-				// $msg[] = 'Exception: ' . $e->getMessage() . ' at ' . $e->getFile() . ':' . $e->getLine();
-				// $msg[] = '';
-				// $msg[] = $e->getTraceAsString();
-				// echo implode("\n", $msg);
-				// echo "\n";
 			}
 		});
 
@@ -159,49 +166,63 @@ abstract class ApiController extends Controller {
 		$output = array();
 		$input = file_get_contents('php://input');
 
+		//var_dump($input, $_SERVER['CONTENT_TYPE']);die();
+
 		// Two content types are supported:
 		//
 		// multipart/form-data; boundary=------------------------68670b1a1565e787
 		// application/x-www-form-urlencoded
 
-		if (!isset($_SERVER['CONTENT_TYPE']) || $_SERVER['CONTENT_TYPE'] == 'application/x-www-form-urlencoded') {
+		if (!isset($_SERVER['CONTENT_TYPE']) || strpos($_SERVER['CONTENT_TYPE'], 'application/x-www-form-urlencoded') === 0) {
 			parse_str($input, $output);
 		} else {
-			if (!isset($_SERVER['CONTENT_TYPE'])) throw new \Exception("Cannot decode input data");
+			throw new \Exception('Only application/x-www-form-urlencoded Content-Type is supported');
-			preg_match('/boundary=(.*)$/', $_SERVER['CONTENT_TYPE'], $matches);
-			if (!isset($matches[1])) throw new \Exception("Cannot decode input data");
-			$boundary = $matches[1];
-			$blocks = preg_split("/-+$boundary/", $input);
-			array_pop($blocks);
-			foreach ($blocks as $id => $block) {
-				if (empty($block)) continue;
 
-				// you'll have to var_dump $block to understand this and maybe replace \n or \r with a visibile char
+			// if (!isset($_SERVER['CONTENT_TYPE'])) throw new \Exception("Cannot decode input data");
+			// preg_match('/boundary=(.*)$/', $_SERVER['CONTENT_TYPE'], $matches);
+			// if (!isset($matches[1])) throw new \Exception("Cannot decode input data");
 
-				// parse uploaded files
+			// $boundary = $matches[1];
-				if (strpos($block, 'application/octet-stream') !== FALSE) {
+			// $lines = explode("\r\n", $input);
-					// match "name", then everything after "stream" (optional) except for prepending newlines 
-					preg_match("/name=\"([^\"]*)\".*stream[\n|\r]+([^\n\r].*)?$/s", $block, $matches);
-				} else {
-					// match "name" and optional value in between newline sequences
-					preg_match('/name=\"([^\"]*)\"[\n|\r]+([^\n\r].*)?\r$/s', $block, $matches);
-				}
-				if (!isset($matches[2])) {
-					// Regex above will not find anything if the parameter has no value. For example
-					// "parent_id" below:
 
-					// Content-Disposition: form-data; name="parent_id"
+			// $state = 'out';
-					//
-					//
-					// Content-Disposition: form-data; name="id"
-					//
-					// 54ad197be333c98778c7d6f49506efcb
 
-					$output[$matches[1]] = '';
+			// foreach ($lines as $line) {
-				} else {
+
-					$output[$matches[1]] = $matches[2];
+			// }
-				}
+
-			}
+			
+			// $blocks = preg_split("/-+$boundary/", $input);
+			// array_pop($blocks);
+			// foreach ($blocks as $id => $block) {
+			// 	if (empty($block)) continue;
+
+			// 	// you'll have to var_dump $block to understand this and maybe replace \n or \r with a visibile char
+
+			// 	// parse uploaded files
+			// 	if (strpos($block, 'application/octet-stream') !== FALSE) {
+			// 		// match "name", then everything after "stream" (optional) except for prepending newlines 
+			// 		preg_match("/name=\"([^\"]*)\".*stream[\n|\r]+([^\n\r].*)?$/s", $block, $matches);
+			// 	} else {
+			// 		// match "name" and optional value in between newline sequences
+			// 		preg_match('/name=\"([^\"]*)\"[\n|\r]+([^\n\r].*)?\r$/s', $block, $matches);
+			// 	}
+			// 	if (!isset($matches[2])) {
+			// 		// Regex above will not find anything if the parameter has no value. For example
+			// 		// "parent_id" below:
+
+			// 		// Content-Disposition: form-data; name="parent_id"
+			// 		//
+			// 		//
+			// 		// Content-Disposition: form-data; name="id"
+			// 		//
+			// 		// 54ad197be333c98778c7d6f49506efcb
+
+			// 		$output[$matches[1]] = '';
+			// 	} else {
+			// 		$output[$matches[1]] = $matches[2];
+			// 	}
+			// }
 		}
 
 		return $output;

src/AppBundle/Controller/FoldersController.php

@@ -56,6 +56,7 @@ class FoldersController extends ApiController {
 		if ($request->isMethod('PATCH')) {
 			$data = $this->patchParameters();
 			$folder->fromPublicArray($this->patchParameters());
+			$folder->id = Folder::unhex($id);
 			$folder->save();
 			return static::successResponse($folder);
 		}