self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-04-14 11:18:47 +02:00
Code Issues Releases Activity

Synchronizer

Browse Source
This commit is contained in:
Laurent Cozic 2017-05-19 19:12:09 +00:00
parent 8cce2af07e
commit a3d2c9819e
12 changed files with 300 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
ReactNativeClient/src/base-model.js
View File

@ -58,13 +58,17 @@ class BaseModel {
return options;
}
static load(id) {
return this.db().selectOne('SELECT * FROM ' + this.tableName() + ' WHERE id = ?', [id]);
}
static saveQuery(o, isNew = 'auto') {
if (isNew == 'auto') isNew = !o.id;
let query = '';
let itemId = o.id;
if (isNew) {
if (this.useUuid()) {
if (this.useUuid() && !o.id) {
o = Object.assign({}, o);
itemId = uuid.create();
o.id = itemId;
@ -96,15 +100,28 @@ class BaseModel {
// which are not handled by React Native.
const { Change } = require('src/models/change.js');
let change = Change.newChange();
change.type = isNew ? Change.TYPE_CREATE : Change.TYPE_UPDATE;
change.item_id = query.id;
change.item_type = this.itemType();
if (isNew) {
let change = Change.newChange();
change.type = Change.TYPE_CREATE;
change.item_id = query.id;
change.item_type = this.itemType();
let changeQuery = Change.saveQuery(change);
tx.executeSql(changeQuery.sql, changeQuery.params);
let changeQuery = Change.saveQuery(change);
tx.executeSql(changeQuery.sql, changeQuery.params);
} else {
for (let n in o) {
if (!o.hasOwnProperty(n)) continue;
// TODO: item field for UPDATE
let change = Change.newChange();
change.type = Change.TYPE_UPDATE;
change.item_id = query.id;
change.item_type = this.itemType();
change.item_field = n;
let changeQuery = Change.saveQuery(change);
tx.executeSql(changeQuery.sql, changeQuery.params);
}
}
}
}).then(() => {
o = Object.assign({}, o);
@ -122,16 +139,16 @@ class BaseModel {
}
return this.db().exec('DELETE FROM ' + this.tableName() + ' WHERE id = ?', [id]).then(() => {
// if (options.trackChanges && this.trackChanges()) {
// const { Change } = require('src/models/change.js');
if (options.trackChanges && this.trackChanges()) {
const { Change } = require('src/models/change.js');
// let change = Change.newChange();
// change.type = Change.TYPE_DELETE;
// change.item_id = id;
// change.item_type = this.itemType();
let change = Change.newChange();
change.type = Change.TYPE_DELETE;
change.item_id = id;
change.item_type = this.itemType();
// return Change.save(change);
// }
return Change.save(change);
}
});
}

2
ReactNativeClient/src/components/item-list.js
View File

@ -72,7 +72,7 @@ class ItemListComponent extends Component {
<TouchableHighlight onPress={onPress} onLongPress={onLongPress}>
<View>
{ isEditable && <Checkbox label={item.title} ></Checkbox> }
{ !isEditable && <Text>{item.title}</Text> }
{ !isEditable && <Text>{item.title} [{item.id}]</Text> }
</View>
</TouchableHighlight>
);

2
ReactNativeClient/src/database.js
View File

@ -110,7 +110,7 @@ class Database {
}
open() {
this.db_ = SQLite.openDatabase({ name: '/storage/emulated/0/Download/joplin-11.sqlite' }, (db) => {
this.db_ = SQLite.openDatabase({ name: '/storage/emulated/0/Download/joplin-12.sqlite' }, (db) => {
Log.info('Database was open successfully');
}, (error) => {
Log.error('Cannot open database: ', error);

1
ReactNativeClient/src/main.js
View File

@ -15,6 +15,7 @@ function main() {
Registry.setDebugMode(true);
AppRegistry.registerComponent('AwesomeProject', () => Root);
Log.setLevel(Registry.debugMode() ? Log.LEVEL_DEBUG : Log.LEVEL_WARN);
Log.info('START ======================================================================================================');
// Note: The final part of the initialization process is in
// AppComponent.componentDidMount(), when the application is ready.
}

89
ReactNativeClient/src/models/change.js
View File

@ -3,7 +3,7 @@ import { Log } from 'src/log.js';
class Change extends BaseModel {
static TYPE_UNKNOWN = 0;
static TYPE_NOOP = 0;
static TYPE_CREATE = 1;
static TYPE_UPDATE = 2;
static TYPE_DELETE = 3;
@ -22,15 +22,84 @@ class Change extends BaseModel {
};
}
// static all() {
// return this.db().selectAll('SELECT * FROM folders').then((r) => {
// let output = [];
// for (let i = 0; i < r.rows.length; i++) {
// output.push(r.rows.item(i));
// }
// return output;
// });
// }
static all() {
return this.db().selectAll('SELECT * FROM changes').then((r) => {
let output = [];
for (let i = 0; i < r.rows.length; i++) {
output.push(r.rows.item(i));
}
return output;
});
}
static deleteMultiple(ids) {
if (ids.length == 0) return Promise.resolve();
return this.db().transaction((tx) => {
let sql = '';
for (let i = 0; i < ids.length; i++) {
tx.executeSql('DELETE FROM changes WHERE id = ?', [ids[i]]);
}
});
}
static mergeChanges(changes) {
let createdItems = [];
let deletedItems = [];
let itemChanges = {};
for (let i = 0; i < changes.length; i++) {
let change = changes[i];
if (itemChanges[change.item_id]) {
mergedChange = itemChanges[change.item_id];
} else {
mergedChange = {
item_id: change.item_id,
item_type: change.item_type,
fields: [],
ids: [],
type: change.type,
}
}
if (change.type == this.TYPE_CREATE) {
createdItems.push(change.item_id);
} else if (change.type == this.TYPE_DELETE) {
deletedItems.push(change.item_id);
} else if (change.type == this.TYPE_UPDATE) {
if (mergedChange.fields.indexOf(change.item_field) < 0) {
mergedChange.fields.push(change.item_field);
}
}
mergedChange.ids.push(change.id);
itemChanges[change.item_id] = mergedChange;
}
let output = [];
for (let itemId in itemChanges) {
if (!itemChanges.hasOwnProperty(itemId)) continue;
let change = itemChanges[itemId];
if (createdItems.indexOf(itemId) >= 0 && deletedItems.indexOf(itemId) >= 0) {
// Item both created then deleted - skip
change.type = this.TYPE_NOOP;
} else if (deletedItems.indexOf(itemId) >= 0) {
// Item was deleted at some point - just return one 'delete' event
change.type = this.TYPE_DELETE;
} else if (createdItems.indexOf(itemId) >= 0) {
// Item was created then updated - just return one 'create' event with the latest changes
change.type = this.TYPE_CREATE;
}
output.push(change);
}
return output;
}
}

61
ReactNativeClient/src/models/setting.js
View File

@ -43,15 +43,11 @@ class Setting extends BaseModel {
}
static setValue(key, value) {
// if (value !== null && typeof value === 'object') {
// return this.setObject(key, value);
// }
this.scheduleUpdate();
for (let i = 0; i < this.cache_.length; i++) {
if (this.cache_[i].key == key) {
if (this.cache_[i].value === value) return;
this.cache_[i].value = value;
this.scheduleUpdate();
return;
}
}
@ -59,19 +55,9 @@ class Setting extends BaseModel {
let s = this.defaultSetting(key);
s.value = value;
this.cache_.push(s);
this.scheduleUpdate();
}
// static del(key) {
// this.scheduleUpdate();
// for (let i = 0; i < this.cache_.length; i++) {
// if (this.cache_[i].key == key) {
// this.cache_[i].value = value;
// return;
// }
// }
// }
static value(key) {
for (let i = 0; i < this.cache_.length; i++) {
if (this.cache_[i].key == key) {
@ -104,23 +90,32 @@ class Setting extends BaseModel {
}
}
static scheduleUpdate() {
if (this.updateTimeoutId) clearTimeout(this.updateTimeoutId);
static saveAll() {
if (!this.updateTimeoutId_) return Promise.resolve();
this.updateTimeoutId = setTimeout(() => {
Log.info('Saving settings...');
this.updateTimeoutId = null;
BaseModel.db().transaction((tx) => {
tx.executeSql('DELETE FROM settings');
for (let i = 0; i < this.cache_.length; i++) {
let q = Database.insertQuery(this.tableName(), this.cache_[i]);
tx.executeSql(q.sql, q.params);
}
}).then(() => {
Log.info('Settings have been saved.');
}).catch((error) => {
Log.warn('Could not update settings:', error);
});
Log.info('Saving settings...');
clearTimeout(this.updateTimeoutId_);
this.updateTimeoutId_ = null;
return BaseModel.db().transaction((tx) => {
tx.executeSql('DELETE FROM settings');
for (let i = 0; i < this.cache_.length; i++) {
let q = Database.insertQuery(this.tableName(), this.cache_[i]);
tx.executeSql(q.sql, q.params);
}
}).then(() => {
Log.info('Settings have been saved.');
}).catch((error) => {
Log.warn('Could not save settings', error);
reject(error);
});
}
static scheduleUpdate() {
if (this.updateTimeoutId_) clearTimeout(this.updateTimeoutId_);
this.updateTimeoutId_ = setTimeout(() => {
this.saveAll();
}, 500);
}

10
ReactNativeClient/src/root.js
View File

@ -32,7 +32,7 @@ let defaultState = {
};
const reducer = (state = defaultState, action) => {
Log.info('Reducer action', action);
Log.info('Reducer action', action.type);
let newState = state;
@ -163,8 +163,8 @@ class AppComponent extends React.Component {
componentDidMount() {
let db = new Database();
db.setDebugEnabled(Registry.debugMode());
//db.setDebugEnabled(Registry.debugMode());
db.setDebugEnabled(false);
BaseModel.dispatch = this.props.dispatch;
db.open().then(() => {
@ -200,8 +200,8 @@ class AppComponent extends React.Component {
Log.warn('Cannot load folders', error);
});
}).then(() => {
// let synchronizer = new Synchronizer();
// synchronizer.start();
let synchronizer = new Synchronizer(db, Registry.api());
synchronizer.start();
}).catch((error) => {
Log.error('Initialization error:', error);
});

80
ReactNativeClient/src/synchronizer.js
View File

@ -3,11 +3,14 @@ import { Log } from 'src/log.js';
import { Setting } from 'src/models/setting.js';
import { Change } from 'src/models/change.js';
import { Folder } from 'src/models/folder.js';
import { promiseChain } from 'src/promise-chain.js';
class Synchronizer {
constructor() {
constructor(db, api) {
this.state_ = 'idle';
this.db_ = db;
this.api_ = api;
}
state() {
@ -15,52 +18,105 @@ class Synchronizer {
}
db() {
return Registry.db();
return this.db_;
}
api() {
return Registry.api();
return this.api_;
}
switchState(state) {
Log.info('Sync: switching state to: ' + state);
if (state == 'downloadChanges') {
let maxRevId = null;
this.api().get('synchronizer', { last_id: Setting.value('sync.lastRevId') }).then((syncOperations) => {
let promise = new Promise((resolve, reject) => { resolve(); });
let chain = [];
for (let i = 0; i < syncOperations.items.length; i++) {
let syncOp = syncOperations.items[i];
if (syncOp.id > maxRevId) maxRevId = syncOp.id;
if (syncOp.item_type == 'folder') {
if (syncOp.type == 'create') {
promise = promise.then(() => {
chain.push(() => {
let folder = Folder.fromApiResult(syncOp.item);
// TODO: automatically handle NULL fields by checking type and default value of field
if (!folder.parent_id) folder.parent_id = '';
return Folder.save(folder, { isNew: true });
});
}
// TODO: update
// TODO: delete
}
}
return promiseChain(chain);
}).then(() => {
Log.info('All items synced.');
if (maxRevId) {
Setting.setValue('sync.lastRevId', maxRevId);
return Setting.saveAll();
}
}).then(() => {
this.switchState('uploadingChanges');
}).catch((error) => {
Log.warn('Sync error', error);
});
} else if (state == 'uploadingChanges') {
Change.all().then((changes) => {
let mergedChanges = Change.mergeChanges(changes);
// Log.info(mergedChanges);
let chain = [];
let processedChangeIds = [];
for (let i = 0; i < mergedChanges.length; i++) {
let c = mergedChanges[i];
chain.push(() => {
let p = null;
promise.then(() => {
Log.info('All items synced.');
}).catch((error) => {
Log.warn('Sync error', error);
Log.info(this.api());
if (c.type == Change.TYPE_NOOP) {
p = Promise.resolve();
} else if (c.type == Change.TYPE_CREATE) {
p = Folder.load(c.item_id).then((folder) => {
return this.api().put('folders/' + folder.id, null, folder);
});
} else if (c.type == Change.TYPE_UPDATE) {
p = Folder.load(c.item_id).then((folder) => {
return this.api().patch('folders/' + folder.id, null, folder);
});
} else if (c.type == Change.TYPE_DELETE) {
p = Folder.load(c.item_id).then((folder) => {
return this.api().delete('folders/' + folder.id);
});
}
return p.then(() => {
processedChangeIds = processedChangeIds.concat(c.ids);
});
});
}
promiseChain(chain).then(() => {
Log.info('IDs to delete: ', processedChangeIds);
Change.deleteMultiple(processedChangeIds);
});
});
} else {
}
}
start() {
Log.info('Sync: start');
if (this.state() != 'idle') {
Log.info("Sync: cannot start synchronizer because synchronization already in progress. State: " + this.state());
return;
}
Log.info('Sync: start');
if (!this.api().session()) {
Log.info("Sync: cannot start synchronizer because user is not logged in.");
return;
}
this.switchState('downloadChanges');
}

23
ReactNativeClient/src/web-api.js
View File

@ -23,11 +23,18 @@ class WebApi {
let options = {};
options.method = method.toUpperCase();
if (data) {
var formData = new FormData();
for (var key in data) {
if (!data.hasOwnProperty(key)) continue;
formData.append(key, data[key]);
let formData = null;
if (method == 'POST') {
formData = new FormData();
for (var key in data) {
if (!data.hasOwnProperty(key)) continue;
formData.append(key, data[key]);
}
} else {
options.headers = { 'Content-Type': 'application/x-www-form-urlencoded' };
formData = stringify(data);
}
options.body = formData;
}
@ -91,6 +98,14 @@ class WebApi {
return this.exec('POST', path, query, data);
}
put(path, query, data) {
return this.exec('PUT', path, query, data);
}
patch(path, query, data) {
return this.exec('PATCH', path, query, data);
}
delete(path, query) {
return this.exec('DELETE', path, query);
}

105
src/AppBundle/Controller/ApiController.php
View File

@ -32,24 +32,31 @@ abstract class ApiController extends Controller {
$r->send();
echo "\n";
} else {
$msg = $e->getMessage();
// If the message was sent in Latin encoding, JsonResponse below will fail
// so encode it using UTF-8 here.
if (json_encode($msg) === false) {
$msg = utf8_encode($e->getMessage());
}
$r = array(
'error' => $e->getMessage(),
'error' => $msg,
'code' => 0,
'type' => 'Exception',
//'trace' => $e->getTraceAsString(),
);
$response = new JsonResponse($r);
try {
$response = new JsonResponse($r);
} catch (\Exception $wat) {
// If that happens, print the error message as is, since it's better than showing nothing at all
die($e->getMessage());
}
$response->setStatusCode(500);
$response->send();
echo "\n";
// $msg = array();
// $msg[] = 'Exception: ' . $e->getMessage() . ' at ' . $e->getFile() . ':' . $e->getLine();
// $msg[] = '';
// $msg[] = $e->getTraceAsString();
// echo implode("\n", $msg);
// echo "\n";
}
});
@ -159,49 +166,63 @@ abstract class ApiController extends Controller {
$output = array();
$input = file_get_contents('php://input');
//var_dump($input, $_SERVER['CONTENT_TYPE']);die();
// Two content types are supported:
//
// multipart/form-data; boundary=------------------------68670b1a1565e787
// application/x-www-form-urlencoded
if (!isset($_SERVER['CONTENT_TYPE']) || $_SERVER['CONTENT_TYPE'] == 'application/x-www-form-urlencoded') {
if (!isset($_SERVER['CONTENT_TYPE']) || strpos($_SERVER['CONTENT_TYPE'], 'application/x-www-form-urlencoded') === 0) {
parse_str($input, $output);
} else {
if (!isset($_SERVER['CONTENT_TYPE'])) throw new \Exception("Cannot decode input data");
preg_match('/boundary=(.*)$/', $_SERVER['CONTENT_TYPE'], $matches);
if (!isset($matches[1])) throw new \Exception("Cannot decode input data");
$boundary = $matches[1];
$blocks = preg_split("/-+$boundary/", $input);
array_pop($blocks);
foreach ($blocks as $id => $block) {
if (empty($block)) continue;
throw new \Exception('Only application/x-www-form-urlencoded Content-Type is supported');
// you'll have to var_dump $block to understand this and maybe replace \n or \r with a visibile char
// if (!isset($_SERVER['CONTENT_TYPE'])) throw new \Exception("Cannot decode input data");
// preg_match('/boundary=(.*)$/', $_SERVER['CONTENT_TYPE'], $matches);
// if (!isset($matches[1])) throw new \Exception("Cannot decode input data");
// parse uploaded files
if (strpos($block, 'application/octet-stream') !== FALSE) {
// match "name", then everything after "stream" (optional) except for prepending newlines
preg_match("/name=\"([^\"]*)\".*stream[\n|\r]+([^\n\r].*)?$/s", $block, $matches);
} else {
// match "name" and optional value in between newline sequences
preg_match('/name=\"([^\"]*)\"[\n|\r]+([^\n\r].*)?\r$/s', $block, $matches);
}
if (!isset($matches[2])) {
// Regex above will not find anything if the parameter has no value. For example
// "parent_id" below:
// $boundary = $matches[1];
// $lines = explode("\r\n", $input);
// Content-Disposition: form-data; name="parent_id"
//
//
// Content-Disposition: form-data; name="id"
//
// 54ad197be333c98778c7d6f49506efcb
// $state = 'out';
$output[$matches[1]] = '';
} else {
$output[$matches[1]] = $matches[2];
}
}
// foreach ($lines as $line) {
// }
// $blocks = preg_split("/-+$boundary/", $input);
// array_pop($blocks);
// foreach ($blocks as $id => $block) {
// if (empty($block)) continue;
// // you'll have to var_dump $block to understand this and maybe replace \n or \r with a visibile char
// // parse uploaded files
// if (strpos($block, 'application/octet-stream') !== FALSE) {
// // match "name", then everything after "stream" (optional) except for prepending newlines
// preg_match("/name=\"([^\"]*)\".*stream[\n|\r]+([^\n\r].*)?$/s", $block, $matches);
// } else {
// // match "name" and optional value in between newline sequences
// preg_match('/name=\"([^\"]*)\"[\n|\r]+([^\n\r].*)?\r$/s', $block, $matches);
// }
// if (!isset($matches[2])) {
// // Regex above will not find anything if the parameter has no value. For example
// // "parent_id" below:
// // Content-Disposition: form-data; name="parent_id"
// //
// //
// // Content-Disposition: form-data; name="id"
// //
// // 54ad197be333c98778c7d6f49506efcb
// $output[$matches[1]] = '';
// } else {
// $output[$matches[1]] = $matches[2];
// }
// }
}
return $output;

1
src/AppBundle/Controller/FoldersController.php
View File

@ -56,6 +56,7 @@ class FoldersController extends ApiController {
if ($request->isMethod('PATCH')) {
$data = $this->patchParameters();
$folder->fromPublicArray($this->patchParameters());
$folder->id = Folder::unhex($id);
$folder->save();
return static::successResponse($folder);
}

2
src/AppBundle/Model/BaseModel.php
View File

@ -373,7 +373,7 @@ class BaseModel extends \Illuminate\Database\Eloquent\Model {
unset($changedFields['updated_time']);
}
$output = parent::save($options);
$output = parent::save($options);
//$this->cacheClear();