self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-27 10:32:58 +02:00
Code Issues Releases Activity

Sync fuzzing

Browse Source
This commit is contained in:
Laurent Cozic 2017-06-30 23:53:22 +01:00
parent dc4982e17c
commit 51e38b4023
10 changed files with 543 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CliClient/.gitignore vendored
View File

@ -5,6 +5,7 @@ tests-build/
tests/src tests/src
config.json config.json
app/lib app/lib
tests/fuzzing/client0
tests/fuzzing/client1 tests/fuzzing/client1
tests/fuzzing/client2 tests/fuzzing/client2
tests/fuzzing/sync tests/fuzzing/sync

208
CliClient/app/fuzzing.js Normal file
View File

@ -0,0 +1,208 @@
require('source-map-support').install();
require('babel-plugin-transform-runtime');
import { time } from 'lib/time-utils.js';
import { Logger } from 'lib/logger.js';
import lodash from 'lodash';
const exec = require('child_process').exec
const fs = require('fs-extra');
const baseDir = '/var/www/joplin/CliClient/tests/fuzzing';
const syncDir = baseDir + '/sync';
const joplinAppPath = __dirname + '/main.js';
const logger = new Logger();
logger.addTarget('console');
logger.setLevel(Logger.LEVEL_DEBUG);
function createClient(id) {
return {
'id': id,
'profileDir': baseDir + '/client' + id,
};
}
async function createClients() {
let output = [];
let promises = [];
for (let clientId = 0; clientId < 2; clientId++) {
let client = createClient(clientId);
promises.push(fs.remove(client.profileDir));
promises.push(execCommand(client, 'config sync.target local').then(() => { return execCommand(client, 'config sync.local.path ' + syncDir); }));
output.push(client);
}
await Promise.all(promises);
return output;
}
function randomElement(array) {
if (!array.length) return null;
return array[Math.floor(Math.random() * array.length)];
}
function randomWord() {
const words = ['future','breezy','north','untidy','welcome','tenuous','material','tour','erect','bounce','skirt','compare','needle','abstracted','flower','detect','market','boring','lively','ragged','many','safe','credit','periodic','fold','whip','lewd','perform','nonchalant','rigid','amusing','giant','slippery','dog','tranquil','ajar','fanatical','flood','learned','helpless','size','ambiguous','long','six','jealous','history','distance','automatic','soggy','statuesque','prevent','full','price','parallel','mine','garrulous','wandering','puzzled','argument','sack','boil','marked','alive','observe','earsplitting','loving','fallacious','ice','parched','gleaming','horse','frame','gorgeous','quartz','quill','found','stranger','digestion','balance','cut','savory','peace','passenger','driving','sand','offer','rightful','earthquake','ear','spark','seashore','godly','rabbits','time','flowers','womanly','sulky','penitent','detail','warm','functional','silver','bushes','veil','filthy','jar','stitch','heartbreaking','bite-sized','station','play','plastic','common','save','subsequent','miscreant','slimy','train','disgusted','new','crib','boundless','stop','zephyr','roof','boiling','humdrum','record','park','symptomatic','vegetable','interest','ring','dusty','pet','depressed','murder','humor','capricious','kiss','gold','fax','cycle','river','black','four','irritating','mature','well-groomed','guard','hand','spotty','celery','air','scent','jelly','alleged','preach','anger','daffy','wrestle','torpid','excuse','jump','paint','exotic','tasty','auspicious','shirt','exercise','planes','romantic','telephone','teaching','towering','line','grouchy','eggnog','treat','powerful','abortive','paddle','belief','smash','fowl','steam','scale','workable','overwrought','elated','rustic','cuddly','star','extra-small','wacky','marry','optimal','muddle','care','turn','wealthy','phobic','ticket','petite','order','curly','lazy','careful','unequaled','mountain','attract','guide','robin','plant','hook','sail','creature','sparkle','sugar','volcano','grate','plough','undesirable','clever','mark','sea','responsible','destroy','broken','bore','spell','gate','lean','eye','afternoon','grease','note','smiling','puzzling','annoy','disagreeable','valuable','judge','frequent','live','gentle','reward','calm','aloof','old-fashioned','rule','sweet','hat','lumber','cheer','writing','able','roasted','scream','awful','meaty','nutty','trade','protest','letter','half','spiteful','library','food','sign','side','adhesive','itch','fuzzy','force','circle','historical','door','behavior','smile','bitter','scatter','crow','risk','rebel','milky','wise','rule','confuse','motion','roll','grain','structure','ship','admire','discreet','test','ask','meddle','tacit','abundant','skin','wound','beds','saw','few','rhyme','heavenly','jaded','finger','advice','letters','satisfying','general','add','fork','impartial','remind','rate','rotten','beam','puffy','march','horn','practise','brief','coordinated','ahead','woebegone','insidious','continue','rapid','adamant','gray','bless','dinosaurs','dress','woman','stir','songs','unwieldy','jump','cows','dust','terrify','acrid','illegal','desire','share','strange','damaged','entertaining','stare','underwear','legal','oven','refuse','accidental','blot','snakes','talk','lunchroom','man','blushing','waste','aggressive','oval','tax','clam','present','important','chicken','name','town','mend','knowing','long','wrathful','kettle','difficult','account','choke','decorate','bead','fear','majestic','shame','laborer','wine','story','hissing','stingy','plant','potato','houses','leg','number','condemned','hollow','bashful','distinct','ray','evanescent','whimsical','magic','bomb','cute','omniscient','plane','immense','brake','time','marvelous','mask','conscious','explain','answer','physical','berry','guide','machine','toad','business','milk','examine','chickens','uppity','red','kind','medical','shiver','punch','lake','sleepy','axiomatic','matter','nosy','zealous','mint','embarrassed','psychedelic','imagine','collar','tame','wing','soup','efficient','rat','signal','delight','belong','ducks','wicked','nod','close','snotty','measly','front','flag','smoke','magenta','squash','bubble','downtown','thirsty','tremendous','closed','stupid','shaggy','receipt','low','famous','momentous','grateful','concerned','tart','bomb','existence','vacation','grandfather','duck','bubble','reason','glue','assorted','peaceful','questionable','type','industry','chemical','rambunctious','plant','heap','church','suggestion','tickle','income','aberrant','enormous','knock'];
return randomElement(words);
}
function execCommand(client, command) {
let exePath = 'node ' + joplinAppPath;
let cmd = exePath + ' --profile ' + client.profileDir + ' ' + command;
//logger.info(cmd.substr(exePath.length + 1));
logger.info(cmd);
return new Promise((resolve, reject) => {
exec(cmd, (error, stdout, stderr) => {
if (error) {
logger.error(stderr);
reject(error);
} else {
resolve(stdout);
}
});
});
}
async function execRandomCommand(client) {
let possibleCommands = [
['mkbook {word}', 30],
['mknote {word}', 100],
[async () => {
let items = await execCommand(client, 'dump');
items = JSON.parse(items);
let item = randomElement(items);
if (!item) return;
if (item.type_ == 1) {
await execCommand(client, 'rm -f ' + item.title);
} else if (item.type_ == 2) {
await execCommand(client, 'rm -f ' + '../' + item.title);
} else {
throw new Error('Unknown type: ' + item.type_);
}
}, 40],
['sync', 10],
];
let cmd = null;
while (true) {
cmd = randomElement(possibleCommands);
let r = 1 + Math.floor(Math.random() * 100);
if (r <= cmd[1]) break;
}
cmd = cmd[0];
if (typeof cmd === 'function') {
return cmd();
} else {
cmd = cmd.replace('{word}', randomWord());
return execCommand(client, cmd);
}
}
function randomNextCheckTime() {
let output = time.unixMs() + 1000 + Math.random() * 1000 * 10;
logger.info('Next sync check: ' + time.unixMsToIso(output) + ' (' + (Math.round((output - time.unixMs()) / 1000)) + ' sec.)');
return output;
}
async function compareClientItems(clientItems) {
let itemCounts = [];
for (let i = 0; i < clientItems.length; i++) {
let items = clientItems[i];
itemCounts.push(items.length);
}
logger.info('Item count: ' + itemCounts.join(', '));
let r = lodash.uniq(itemCounts);
if (r.length > 1) {
logger.error('Item count is different');
await time.sleep(2); // Let the logger finish writing
process.exit(1);
}
}
async function main(argv) {
await fs.remove(syncDir);
let clients = await createClients();
let activeCommandCounts = [];
let clientId = 0;
for (let i = 0; i < clients.length; i++) {
clients[i].activeCommandCount = 0;
}
function handleCommand(clientId) {
if (clients[clientId].activeCommandCount >= 1) return;
clients[clientId].activeCommandCount++;
execRandomCommand(clients[clientId]).catch((error) => {
logger.info('Client ' + clientId + ':');
logger.error(error);
}).then((r) => {
if (r) {
logger.info('Client ' + clientId + ':');
logger.info(r);
}
clients[clientId].activeCommandCount--;
});
}
let nextSyncCheckTime = randomNextCheckTime();
let state = 'commands';
setInterval(async () => {
if (state == 'waitForSyncCheck') return;
if (state == 'syncCheck') {
state = 'waitForSyncCheck';
let clientItems = [];
// In order for all the clients to send their items and get those from the other
// clients, they need to perform 2 sync.
for (let loopCount = 0; loopCount < 2; loopCount++) {
for (let i = 0; i < clients.length; i++) {
await execCommand(clients[i], 'sync');
if (loopCount === 1) {
let dump = await execCommand(clients[i], 'dump');
clientItems[i] = JSON.parse(dump);
}
}
}
await compareClientItems(clientItems);
nextSyncCheckTime = randomNextCheckTime();
state = 'commands';
return;
}
if (state == 'waitForClients') {
for (let i = 0; i < clients.length; i++) {
if (clients[i].activeCommandCount > 0) return;
}
state = 'syncCheck';
return;
}
if (state == 'commands') {
if (nextSyncCheckTime <= time.unixMs()) {
state = 'waitForClients';
return;
}
handleCommand(clientId);
clientId++;
if (clientId >= clients.length) clientId = 0;
}
}, 100);
}
main(process.argv);

42
CliClient/app/main.js
View File

@ -182,10 +182,14 @@ commands.push({
commands.push({ commands.push({
usage: 'rm <pattern>', usage: 'rm <pattern>',
description: 'Deletes the given item. For a notebook, all the notes within that notebook will be deleted. Use `rm ../<notebook>` to delete a notebook.', description: 'Deletes the given item. For a notebook, all the notes within that notebook will be deleted. Use `rm ../<notebook>` to delete a notebook.',
options: [
['-f, --force', 'Deletes the items without asking for confirmation.'],
],
action: async function(args, end) { action: async function(args, end) {
try { try {
let pattern = args['pattern']; let pattern = args['pattern'];
let itemType = null; let itemType = null;
let force = args.options && args.options.force === true;
if (pattern.indexOf('*') < 0) { // Handle it as a simple title if (pattern.indexOf('*') < 0) { // Handle it as a simple title
if (pattern.substr(0, 3) == '../') { if (pattern.substr(0, 3) == '../') {
@ -197,16 +201,19 @@ commands.push({
let item = await BaseItem.loadItemByField(itemType, 'title', pattern); let item = await BaseItem.loadItemByField(itemType, 'title', pattern);
if (!item) throw new Error(_('No item with title "%s" found.', pattern)); if (!item) throw new Error(_('No item with title "%s" found.', pattern));
await BaseItem.deleteItem(itemType, item.id);
let ok = force ? true : await cmdPromptConfirm(this, _('Delete item?'));
if (ok) {
await BaseItem.deleteItem(itemType, item.id);
if (currentFolder && currentFolder.id == item.id) { if (currentFolder && currentFolder.id == item.id) {
let f = await Folder.defaultFolder(); let f = await Folder.defaultFolder();
switchCurrentFolder(f); switchCurrentFolder(f);
} }
}
} else { // Handle it as a glob pattern } else { // Handle it as a glob pattern
let notes = await Note.previews(currentFolder.id, { titlePattern: pattern }); let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern)); if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern));
let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length)); let ok = force ? true : await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length));
if (ok) { if (ok) {
for (let i = 0; i < notes.length; i++) { for (let i = 0; i < notes.length; i++) {
await Note.delete(notes[i].id); await Note.delete(notes[i].id);
@ -246,6 +253,31 @@ commands.push({
autocomplete: autocompleteItems, autocomplete: autocompleteItems,
}); });
commands.push({
usage: 'dump',
description: 'Dumps the complete database as JSON.',
action: async function(args, end) {
try {
let items = [];
let folders = await Folder.all();
for (let i = 0; i < folders.length; i++) {
let folder = folders[i];
let notes = await Note.previews(folder.id);
items.push(folder);
console.info(folder.title);
items = items.concat(notes);
}
this.log(JSON.stringify(items));
} catch (error) {
this.log(error);
}
end();
},
autocomplete: autocompleteFolders,
});
commands.push({ commands.push({
usage: 'ls [pattern]', usage: 'ls [pattern]',
description: 'Displays the notes in [notebook]. Use `ls ..` to display the list of notebooks.', description: 'Displays the notes in [notebook]. Use `ls ..` to display the list of notebooks.',
@ -254,6 +286,7 @@ commands.push({
['-s, --sort <field>', 'Sorts the item by <field> (eg. title, updated_time, created_time).'], ['-s, --sort <field>', 'Sorts the item by <field> (eg. title, updated_time, created_time).'],
['-r, --reverse', 'Reverses the sorting order.'], ['-r, --reverse', 'Reverses the sorting order.'],
['-t, --type <type>', 'Displays only the items of the specific type(s). Can be `n` for notes, `t` for todos, or `nt` for notes and todos (eg. `-tt` would display only the todos, while `-ttd` would display notes and todos.'], ['-t, --type <type>', 'Displays only the items of the specific type(s). Can be `n` for notes, `t` for todos, or `nt` for notes and todos (eg. `-tt` would display only the todos, while `-ttd` would display notes and todos.'],
['-f, --format <format>', 'Either "text" or "json"'],
], ],
action: async function(args, end) { action: async function(args, end) {
try { try {
@ -284,6 +317,9 @@ commands.push({
items = await Note.previews(currentFolder.id, queryOptions); items = await Note.previews(currentFolder.id, queryOptions);
} }
if (options.format && options.format == 'json') {
this.log(JSON.stringify(items));
} else {
for (let i = 0; i < items.length; i++) { for (let i = 0; i < items.length; i++) {
let item = items[i]; let item = items[i];
let line = ''; let line = '';
@ -293,6 +329,7 @@ commands.push({
line += item.title + suffix; line += item.title + suffix;
this.log(line); this.log(line);
} }
}
} catch (error) { } catch (error) {
this.log(error); this.log(error);
} }
@ -774,6 +811,7 @@ async function main() {
let cmd = shellArgsToString(argv); let cmd = shellArgsToString(argv);
await vorpal.exec(cmd); await vorpal.exec(cmd);
await vorpal.exec('exit'); await vorpal.exec('exit');
await time.sleep(1); // Let loggers finish writing
return; return;
} else { } else {
vorpal.delimiter(promptString()).show(); vorpal.delimiter(promptString()).show();

2
CliClient/package.json
View File

@ -7,7 +7,7 @@
"url": "https://github.com/laurent22/joplin" "url": "https://github.com/laurent22/joplin"
}, },
"url": "git://github.com/laurent22/joplin.git", "url": "git://github.com/laurent22/joplin.git",
"version": "0.8.21", "version": "0.8.27",
"bin": { "bin": {
"joplin": "./main.sh" "joplin": "./main.sh"
}, },

2
CliClient/run_test.sh
View File

@ -7,5 +7,5 @@ ln -s "$CLIENT_DIR/build/lib" "$CLIENT_DIR/tests-build"
npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js tests-build/base-model.js npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js tests-build/base-model.js
#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js
#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js #npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js
#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/models/folder.js

71
CliClient/tests/fuzzing/fuzzing.php
View File

@ -1,71 +0,0 @@
<?php
$clientId = isset($argv[1]) ? $argv[1] : null;
if (!$clientId) throw new Exception('Client ID not set');
function createClient($id) {
return array(
'id' => $id,
'profileDir' => dirname(__FILE__) . '/client' . $id,
);
}
$client = createClient($clientId);
function deltree($dir) {
$files = array_diff(scandir($dir), array('.','..'));
foreach ($files as $file) {
is_dir("$dir/$file") ? deltree("$dir/$file") : unlink("$dir/$file");
}
return rmdir($dir);
}
function randomElement($array) {
return $array[mt_rand(0, count($array) - 1)];
}
function randomWord() {
$words = array('future','breezy','north','untidy','welcome','tenuous','material','tour','erect','bounce','skirt','compare','needle','abstracted','flower','detect','market','boring','lively','ragged','many','safe','credit','periodic','fold','whip','lewd','perform','nonchalant','rigid','amusing','giant','slippery','dog','tranquil','ajar','fanatical','flood','learned','helpless','size','ambiguous','long','six','jealous','history','distance','automatic','soggy','statuesque','prevent','full','price','parallel','mine','garrulous','wandering','puzzled','argument','sack','boil','marked','alive','observe','earsplitting','loving','fallacious','ice','parched','gleaming','horse','frame','gorgeous','quartz','quill','found','stranger','digestion','balance','cut','savory','peace','passenger','driving','sand','offer','rightful','earthquake','ear','spark','seashore','godly','rabbits','time','flowers','womanly','sulky','penitent','detail','warm','functional','silver','bushes','veil','filthy','jar','stitch','heartbreaking','bite-sized','station','play','plastic','common','save','subsequent','miscreant','slimy','train','disgusted','new','crib','boundless','stop','zephyr','roof','boiling','humdrum','record','park','symptomatic','vegetable','interest','ring','dusty','pet','depressed','murder','humor','capricious','kiss','gold','fax','cycle','river','black','four','irritating','mature','well-groomed','guard','hand','spotty','celery','air','scent','jelly','alleged','preach','anger','daffy','wrestle','torpid','excuse','jump','paint','exotic','tasty','auspicious','shirt','exercise','planes','romantic','telephone','teaching','towering','line','grouchy','eggnog','treat','powerful','abortive','paddle','belief','smash','fowl','steam','scale','workable','overwrought','elated','rustic','cuddly','star','extra-small','wacky','marry','optimal','muddle','care','turn','wealthy','phobic','ticket','petite','order','curly','lazy','careful','unequaled','mountain','attract','guide','robin','plant','hook','sail','creature','sparkle','sugar','volcano','grate','plough','undesirable','clever','mark','sea','responsible','destroy','broken','bore','spell','gate','lean','eye','afternoon','grease','note','smiling','puzzling','annoy','disagreeable','valuable','judge','frequent','live','gentle','reward','calm','aloof','old-fashioned','rule','sweet','hat','lumber','cheer','writing','able','roasted','scream','awful','meaty','nutty','trade','protest','letter','half','spiteful','library','food','sign','side','adhesive','itch','fuzzy','force','circle','historical','door','behavior','smile','bitter','scatter','crow','risk','rebel','milky','wise','rule','confuse','motion','roll','grain','structure','ship','admire','discreet','test','ask','meddle','tacit','abundant','skin','wound','beds','saw','few','rhyme','heavenly','jaded','finger','advice','letters','satisfying','general','add','fork','impartial','remind','rate','rotten','beam','puffy','march','horn','practise','brief','coordinated','ahead','woebegone','insidious','continue','rapid','adamant','gray','bless','dinosaurs','dress','woman','stir','songs','unwieldy','jump','cows','dust','terrify','acrid','illegal','desire','share','strange','damaged','entertaining','stare','underwear','legal','oven','refuse','accidental','blot','snakes','talk','lunchroom','man','blushing','waste','aggressive','oval','tax','clam','present','important','chicken','name','town','mend','knowing','long','wrathful','kettle','difficult','account','choke','decorate','bead','fear','majestic','shame','laborer','wine','story','hissing','stingy','plant','potato','houses','leg','number','condemned','hollow','bashful','distinct','ray','evanescent','whimsical','magic','bomb','cute','omniscient','plane','immense','brake','time','marvelous','mask','conscious','explain','answer','physical','berry','guide','machine','toad','business','milk','examine','chickens','uppity','red','kind','medical','shiver','punch','lake','sleepy','axiomatic','matter','nosy','zealous','mint','embarrassed','psychedelic','imagine','collar','tame','wing','soup','efficient','rat','signal','delight','belong','ducks','wicked','nod','close','snotty','measly','front','flag','smoke','magenta','squash','bubble','downtown','thirsty','tremendous','closed','stupid','shaggy','receipt','low','famous','momentous','grateful','concerned','tart','bomb','existence','vacation','grandfather','duck','bubble','reason','glue','assorted','peaceful','questionable','type','industry','chemical','rambunctious','plant','heap','church','suggestion','tickle','income','aberrant','enormous','knock');
return randomElement($words);
}
function execCommand($cmd) {
global $client;
$cmd = 'joplin --profile ' . $client['profileDir'] . ' ' . $cmd;
echo $cmd . "\n";
exec($cmd, $output);
return implode("\n", $output);
}
function execRandomCommand() {
$possibleCommands = array(
array('mkbook {word}', 10),
array('mknote {word}', 100),
array('sync', 10),
);
$cmd = null;
while (true) {
$cmd = randomElement($possibleCommands);
$r = mt_rand(1,100);
if ($r <= $cmd[1]) break;
}
$cmd = $cmd[0];
$cmd = str_replace('{word}', randomWord(), $cmd);
return execCommand($cmd);
}
deltree($client['profileDir']);
execCommand('config sync.target local');
execCommand('config sync.local.path ' . dirname(__FILE__) . '/sync');
while (true) {
$r = execRandomCommand();
if ($r) {
echo $r . "\n";
}
}

438
CliClient/tests/synchronizer.js
View File

@ -23,6 +23,7 @@ async function localItemsSameAsRemote(locals, expect) {
try { try {
let files = await fileApi().list(); let files = await fileApi().list();
files = files.items; files = files.items;
expect(locals.length).toBe(files.length); expect(locals.length).toBe(files.length);
for (let i = 0; i < locals.length; i++) { for (let i = 0; i < locals.length; i++) {
@ -31,6 +32,8 @@ async function localItemsSameAsRemote(locals, expect) {
let remote = await fileApi().stat(path); let remote = await fileApi().stat(path);
expect(!!remote).toBe(true); expect(!!remote).toBe(true);
if (!remote) continue;
expect(remote.updated_time).toBe(dbItem.updated_time); expect(remote.updated_time).toBe(dbItem.updated_time);
let remoteContent = await fileApi().get(path); let remoteContent = await fileApi().get(path);
@ -51,259 +54,248 @@ describe('Synchronizer', function() {
done(); done();
}); });
it('should create remote items', async (done) => { // it('should create remote items', async (done) => {
let folder = await Folder.save({ title: "folder1" }); // let folder = await Folder.save({ title: "folder1" });
await Note.save({ title: "un", parent_id: folder.id }); // await Note.save({ title: "un", parent_id: folder.id });
let all = await allItems(); // let all = await allItems();
await synchronizer().start(); // await synchronizer().start();
await localItemsSameAsRemote(all, expect); // await localItemsSameAsRemote(all, expect);
done(); // done();
}); // });
it('should update remote item', async (done) => { // it('should update remote item', async (done) => {
let folder = await Folder.save({ title: "folder1" }); // let folder = await Folder.save({ title: "folder1" });
let note = await Note.save({ title: "un", parent_id: folder.id }); // let note = await Note.save({ title: "un", parent_id: folder.id });
await synchronizer().start(); // await synchronizer().start();
await sleep(0.1); // await sleep(0.1);
await Note.save({ title: "un UPDATE", id: note.id }); // await Note.save({ title: "un UPDATE", id: note.id });
let all = await allItems(); // let all = await allItems();
await synchronizer().start(); // await synchronizer().start();
await localItemsSameAsRemote(all, expect); // await localItemsSameAsRemote(all, expect);
done(); // done();
}); // });
it('should create local items', async (done) => { // it('should create local items', async (done) => {
let folder = await Folder.save({ title: "folder1" }); // let folder = await Folder.save({ title: "folder1" });
await Note.save({ title: "un", parent_id: folder.id }); // await Note.save({ title: "un", parent_id: folder.id });
await synchronizer().start(); // await synchronizer().start();
await switchClient(2); // await switchClient(2);
await synchronizer().start(); // await synchronizer().start();
let all = await allItems(); // let all = await allItems();
await localItemsSameAsRemote(all, expect); // await localItemsSameAsRemote(all, expect);
done(); // done();
}); // });
it('should update local items', async (done) => { // it('should update local items', async (done) => {
let folder1 = await Folder.save({ title: "folder1" }); // let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: "un", parent_id: folder1.id }); // let note1 = await Note.save({ title: "un", parent_id: folder1.id });
await synchronizer().start(); // await synchronizer().start();
await switchClient(2); // await switchClient(2);
await synchronizer().start(); // await synchronizer().start();
await sleep(0.1); // await sleep(0.1);
let note2 = await Note.load(note1.id); // let note2 = await Note.load(note1.id);
note2.title = "Updated on client 2"; // note2.title = "Updated on client 2";
await Note.save(note2); // await Note.save(note2);
note2 = await Note.load(note2.id); // note2 = await Note.load(note2.id);
await synchronizer().start(); // await synchronizer().start();
await switchClient(1); // await switchClient(1);
await synchronizer().start(); // await synchronizer().start();
let all = await allItems(); // let all = await allItems();
await localItemsSameAsRemote(all, expect); // await localItemsSameAsRemote(all, expect);
done(); // done();
}); // });
it('should resolve note conflicts', async (done) => { // it('should resolve note conflicts', async (done) => {
let folder1 = await Folder.save({ title: "folder1" }); // let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: "un", parent_id: folder1.id }); // let note1 = await Note.save({ title: "un", parent_id: folder1.id });
await synchronizer().start(); // await synchronizer().start();
await switchClient(2); // await switchClient(2);
await synchronizer().start(); // await synchronizer().start();
await sleep(0.1); // await sleep(0.1);
let note2 = await Note.load(note1.id); // let note2 = await Note.load(note1.id);
note2.title = "Updated on client 2"; // note2.title = "Updated on client 2";
await Note.save(note2); // await Note.save(note2);
note2 = await Note.load(note2.id); // note2 = await Note.load(note2.id);
await synchronizer().start(); // await synchronizer().start();
await switchClient(1); // await switchClient(1);
await sleep(0.1); // await sleep(0.1);
let note2conf = await Note.load(note1.id); // let note2conf = await Note.load(note1.id);
note2conf.title = "Updated on client 1"; // note2conf.title = "Updated on client 1";
await Note.save(note2conf); // await Note.save(note2conf);
note2conf = await Note.load(note1.id); // note2conf = await Note.load(note1.id);
await synchronizer().start(); // await synchronizer().start();
let conflictedNotes = await Note.conflictedNotes(); // let conflictedNotes = await Note.conflictedNotes();
expect(conflictedNotes.length).toBe(1); // expect(conflictedNotes.length).toBe(1);
// Other than the id (since the conflicted note is a duplicate), and the is_conflict property // // Other than the id (since the conflicted note is a duplicate), and the is_conflict property
// the conflicted and original note must be the same in every way, to make sure no data has been lost. // // the conflicted and original note must be the same in every way, to make sure no data has been lost.
let conflictedNote = conflictedNotes[0]; // let conflictedNote = conflictedNotes[0];
expect(conflictedNote.id == note2conf.id).toBe(false); // expect(conflictedNote.id == note2conf.id).toBe(false);
for (let n in conflictedNote) { // for (let n in conflictedNote) {
if (!conflictedNote.hasOwnProperty(n)) continue; // if (!conflictedNote.hasOwnProperty(n)) continue;
if (n == 'id' || n == 'is_conflict') continue; // if (n == 'id' || n == 'is_conflict') continue;
expect(conflictedNote[n]).toBe(note2conf[n], 'Property: ' + n); // expect(conflictedNote[n]).toBe(note2conf[n], 'Property: ' + n);
} // }
let noteUpdatedFromRemote = await Note.load(note1.id); // let noteUpdatedFromRemote = await Note.load(note1.id);
for (let n in noteUpdatedFromRemote) { // for (let n in noteUpdatedFromRemote) {
if (!noteUpdatedFromRemote.hasOwnProperty(n)) continue; // if (!noteUpdatedFromRemote.hasOwnProperty(n)) continue;
if (n == 'sync_time') continue; // if (n == 'sync_time') continue;
expect(noteUpdatedFromRemote[n]).toBe(note2[n], 'Property: ' + n); // expect(noteUpdatedFromRemote[n]).toBe(note2[n], 'Property: ' + n);
} // }
done(); // done();
}); // });
it('should resolve folders conflicts', async (done) => { // it('should resolve folders conflicts', async (done) => {
let folder1 = await Folder.save({ title: "folder1" }); // let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: "un", parent_id: folder1.id }); // let note1 = await Note.save({ title: "un", parent_id: folder1.id });
await synchronizer().start(); // await synchronizer().start();
await switchClient(2); // ---------------------------------- // await switchClient(2); // ----------------------------------
await synchronizer().start(); // await synchronizer().start();
await sleep(0.1); // await sleep(0.1);
let folder1_modRemote = await Folder.load(folder1.id); // let folder1_modRemote = await Folder.load(folder1.id);
folder1_modRemote.title = "folder1 UPDATE CLIENT 2"; // folder1_modRemote.title = "folder1 UPDATE CLIENT 2";
await Folder.save(folder1_modRemote); // await Folder.save(folder1_modRemote);
folder1_modRemote = await Folder.load(folder1_modRemote.id); // folder1_modRemote = await Folder.load(folder1_modRemote.id);
await synchronizer().start(); // await synchronizer().start();
await switchClient(1); // ---------------------------------- // await switchClient(1); // ----------------------------------
await sleep(0.1); // await sleep(0.1);
let folder1_modLocal = await Folder.load(folder1.id); // let folder1_modLocal = await Folder.load(folder1.id);
folder1_modLocal.title = "folder1 UPDATE CLIENT 1"; // folder1_modLocal.title = "folder1 UPDATE CLIENT 1";
await Folder.save(folder1_modLocal); // await Folder.save(folder1_modLocal);
folder1_modLocal = await Folder.load(folder1.id); // folder1_modLocal = await Folder.load(folder1.id);
await synchronizer().start(); // await synchronizer().start();
let folder1_final = await Folder.load(folder1.id); // let folder1_final = await Folder.load(folder1.id);
expect(folder1_final.title).toBe(folder1_modRemote.title); // expect(folder1_final.title).toBe(folder1_modRemote.title);
done(); // done();
}); // });
it('should delete remote items', async (done) => { // it('should delete remote notes', async (done) => {
let folder1 = await Folder.save({ title: "folder1" }); // let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: "un", parent_id: folder1.id }); // let note1 = await Note.save({ title: "un", parent_id: folder1.id });
await synchronizer().start(); // await synchronizer().start();
await switchClient(2); // await switchClient(2);
await synchronizer().start(); // await synchronizer().start();
await sleep(0.1); // await sleep(0.1);
await Note.delete(note1.id); // await Note.delete(note1.id);
await synchronizer().start(); // await synchronizer().start();
let files = await fileApi().list(); // let files = await fileApi().list();
files = files.items; // files = files.items;
expect(files.length).toBe(1); // expect(files.length).toBe(1);
expect(files[0].path).toBe(Folder.systemPath(folder1)); // expect(files[0].path).toBe(Folder.systemPath(folder1));
let deletedItems = await BaseModel.deletedItems(); // let deletedItems = await BaseModel.deletedItems();
expect(deletedItems.length).toBe(0); // expect(deletedItems.length).toBe(0);
done(); // done();
}); // });
it('should delete local items', async (done) => { // it('should delete local notes', async (done) => {
let folder1 = await Folder.save({ title: "folder1" }); // let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: "un", parent_id: folder1.id }); // let note1 = await Note.save({ title: "un", parent_id: folder1.id });
await synchronizer().start(); // await synchronizer().start();
await switchClient(2); // await switchClient(2);
await synchronizer().start(); // await synchronizer().start();
await sleep(0.1); // await sleep(0.1);
await Note.delete(note1.id); // await Note.delete(note1.id);
await synchronizer().start(); // await synchronizer().start();
await switchClient(1); // await switchClient(1);
await synchronizer().start(); // await synchronizer().start();
let items = await allItems(); // let items = await allItems();
expect(items.length).toBe(1); // expect(items.length).toBe(1);
let deletedItems = await BaseModel.deletedItems(); // let deletedItems = await BaseModel.deletedItems();
expect(deletedItems.length).toBe(0); // expect(deletedItems.length).toBe(0);
done(); // done();
}); // });
it('should handle conflict when remote note is deleted then local note is modified', async (done) => { // it('should delete remote folder', async (done) => {
let folder1 = await Folder.save({ title: "folder1" }); // let folder1 = await Folder.save({ title: "folder1" });
let note1 = await Note.save({ title: "un", parent_id: folder1.id }); // let folder2 = await Folder.save({ title: "folder2" });
await synchronizer().start(); // await synchronizer().start();
await switchClient(2); // await switchClient(2);
await synchronizer().start(); // await synchronizer().start();
await sleep(0.1); // await sleep(0.1);
await Note.delete(note1.id); // await Folder.delete(folder2.id);
await synchronizer().start(); // await synchronizer().start();
await switchClient(1); // localItemsSameAsRemote();
let newTitle = 'Modified after having been deleted'; // done();
await Note.save({ id: note1.id, title: newTitle }); // });
await synchronizer().start(); it('should delete local folder', async (done) => {
let conflictedNotes = await Note.conflictedNotes();
expect(conflictedNotes.length).toBe(1);
expect(conflictedNotes[0].title).toBe(newTitle);
done();
});
it('should handle conflict when remote folder is deleted then local folder is renamed', async (done) => {
let folder1 = await Folder.save({ title: "folder1" }); let folder1 = await Folder.save({ title: "folder1" });
let folder2 = await Folder.save({ title: "folder2" }); let folder2 = await Folder.save({ title: "folder2" });
let note1 = await Note.save({ title: "un", parent_id: folder1.id });
await synchronizer().start(); await synchronizer().start();
await switchClient(2); await switchClient(2);
@ -312,60 +304,116 @@ describe('Synchronizer', function() {
await sleep(0.1); await sleep(0.1);
await Folder.delete(folder1.id); await Folder.delete(folder2.id);
await synchronizer().start(); await synchronizer().start();
await switchClient(1); await switchClient(1);
await sleep(0.1);
let newTitle = 'Modified after having been deleted';
await Folder.save({ id: folder1.id, title: newTitle });
await synchronizer().start(); await synchronizer().start();
let items = await allItems(); let items = await allItems();
localItemsSameAsRemote(items, expect);
expect(items.length).toBe(1);
done(); done();
}); });
it('should allow duplicate folder title and rename the new one', async (done) => { // it('should handle conflict when remote note is deleted then local note is modified', async (done) => {
let localF1 = await Folder.save({ title: "folder" }); // let folder1 = await Folder.save({ title: "folder1" });
// let note1 = await Note.save({ title: "un", parent_id: folder1.id });
// await synchronizer().start();
await switchClient(2); // await switchClient(2);
let remoteF2 = await Folder.save({ title: "folder" }); // await synchronizer().start();
await synchronizer().start();
await switchClient(1); // await sleep(0.1);
await sleep(0.1); // await Note.delete(note1.id);
await synchronizer().start(); // await synchronizer().start();
let localF2 = await Folder.load(remoteF2.id); // await switchClient(1);
expect(localF2.title == remoteF2.title).toBe(false); // let newTitle = 'Modified after having been deleted';
// await Note.save({ id: note1.id, title: newTitle });
// Then that folder that has been renamed locally should be set in such a way // await synchronizer().start();
// that synchronizing it applies the title change remotely, and that new title
// should be retrieved by client 2.
await synchronizer().start(); // let conflictedNotes = await Note.conflictedNotes();
await switchClient(2); // expect(conflictedNotes.length).toBe(1);
await sleep(0.1); // expect(conflictedNotes[0].title).toBe(newTitle);
await synchronizer().start(); // done();
// });
remoteF2 = await Folder.load(remoteF2.id); // it('should handle conflict when remote folder is deleted then local folder is renamed', async (done) => {
// let folder1 = await Folder.save({ title: "folder1" });
// let folder2 = await Folder.save({ title: "folder2" });
// let note1 = await Note.save({ title: "un", parent_id: folder1.id });
// await synchronizer().start();
expect(remoteF2.title == localF2.title).toBe(true); // await switchClient(2);
done(); // await synchronizer().start();
});
// await sleep(0.1);
// await Folder.delete(folder1.id);
// await synchronizer().start();
// await switchClient(1);
// await sleep(0.1);
// let newTitle = 'Modified after having been deleted';
// await Folder.save({ id: folder1.id, title: newTitle });
// await synchronizer().start();
// let items = await allItems();
// expect(items.length).toBe(1);
// done();
// });
// it('should allow duplicate folder title and rename the new one', async (done) => {
// let localF1 = await Folder.save({ title: "folder" });
// await switchClient(2);
// let remoteF2 = await Folder.save({ title: "folder" });
// await synchronizer().start();
// await switchClient(1);
// await sleep(0.1);
// await synchronizer().start();
// let localF2 = await Folder.load(remoteF2.id);
// expect(localF2.title == remoteF2.title).toBe(false);
// // Then that folder that has been renamed locally should be set in such a way
// // that synchronizing it applies the title change remotely, and that new title
// // should be retrieved by client 2.
// await synchronizer().start();
// await switchClient(2);
// await sleep(0.1);
// await synchronizer().start();
// remoteF2 = await Folder.load(remoteF2.id);
// expect(remoteF2.title == localF2.title).toBe(true);
// done();
// });
}); });

36
lib/logger.js
View File

@ -66,32 +66,34 @@ class Logger {
serializedObject = object; serializedObject = object;
} }
this.fileAppendQueue_.push({ fs.appendFileSync(t.path, line + serializedObject + "\n");
path: t.path,
line: line + serializedObject + "\n",
});
this.scheduleFileAppendQueueProcessing_(); // this.fileAppendQueue_.push({
// path: t.path,
// line: line + serializedObject + "\n",
// });
// this.scheduleFileAppendQueueProcessing_();
} else if (t.type == 'vorpal') { } else if (t.type == 'vorpal') {
t.vorpal.log(object); t.vorpal.log(object);
} }
} }
} }
scheduleFileAppendQueueProcessing_() { // scheduleFileAppendQueueProcessing_() {
if (this.fileAppendQueueTID_) return; // if (this.fileAppendQueueTID_) return;
this.fileAppendQueueTID_ = setTimeout(async () => { // this.fileAppendQueueTID_ = setTimeout(async () => {
this.fileAppendQueueTID_ = null; // this.fileAppendQueueTID_ = null;
let queue = this.fileAppendQueue_.slice(0); // let queue = this.fileAppendQueue_.slice(0);
for (let i = 0; i < queue.length; i++) { // for (let i = 0; i < queue.length; i++) {
let t = queue[i]; // let t = queue[i];
await fs.appendFile(t.path, t.line); // await fs.appendFile(t.path, t.line);
} // }
this.fileAppendQueue_.splice(0, queue.length); // this.fileAppendQueue_.splice(0, queue.length);
}, 10); // }, 1);
} // }
error(object) { return this.log(Logger.LEVEL_ERROR, object); } error(object) { return this.log(Logger.LEVEL_ERROR, object); }
warn(object) { return this.log(Logger.LEVEL_WARN, object); } warn(object) { return this.log(Logger.LEVEL_WARN, object); }

6
lib/models/folder.js
View File

@ -104,7 +104,11 @@ class Folder extends BaseItem {
static save(o, options = null) { static save(o, options = null) {
return Folder.loadByField('title', o.title).then((existingFolder) => { return Folder.loadByField('title', o.title).then((existingFolder) => {
if (existingFolder && existingFolder.id != o.id) throw new Error(_('A notebook with title "%s" already exists', o.title)); if (existingFolder && existingFolder.id != o.id) {
let error = new Error(_('A notebook with title "%s" already exists', o.title));
error.code = 'duplicateTitle';
throw error;
}
return super.save(o, options).then((folder) => { return super.save(o, options).then((folder) => {
this.dispatch({ this.dispatch({

14
lib/synchronizer.js
View File

@ -63,6 +63,7 @@ class Synchronizer {
async logSyncSummary(report) { async logSyncSummary(report) {
for (let n in report) { for (let n in report) {
if (!report.hasOwnProperty(n)) continue;
this.logger().info(n + ': ' + (report[n] ? report[n] : '-')); this.logger().info(n + ': ' + (report[n] ? report[n] : '-'));
} }
let folderCount = await Folder.count(); let folderCount = await Folder.count();
@ -262,7 +263,7 @@ class Synchronizer {
} else { } else {
if (remote.updated_time > local.updated_time) { if (remote.updated_time > local.updated_time) {
action = 'updateLocal'; action = 'updateLocal';
reason = sprintf('remote is more recent than local'); // , time.unixMsToIso(remote.updated_time), time.unixMsToIso(local.updated_time) reason = sprintf('remote is more recent than local');
} }
} }
@ -284,7 +285,18 @@ class Synchronizer {
newContent.sync_time = time.unixMs(); newContent.sync_time = time.unixMs();
let options = { autoTimestamp: false }; let options = { autoTimestamp: false };
if (action == 'createLocal') options.isNew = true; if (action == 'createLocal') options.isNew = true;
try {
await ItemClass.save(newContent, options); await ItemClass.save(newContent, options);
} catch (error) {
if (error.code == 'duplicateTitle') {
newContent.title = newContent.title + '-' + newContent.created_time + '-' + (Math.floor(Math.random() * 1000));
newContent.updated_time = newContent.sync_time + 2;
await ItemClass.save(newContent, options);
} else {
throw error;
}
}
this.logSyncOperation(action, local, content, reason); this.logSyncOperation(action, local, content, reason);
} else { } else {