diff --git a/CliClient/.gitignore b/CliClient/.gitignore index 216a2edff..d5d3a2e06 100644 --- a/CliClient/.gitignore +++ b/CliClient/.gitignore @@ -5,6 +5,7 @@ tests-build/ tests/src config.json app/lib +tests/fuzzing/client0 tests/fuzzing/client1 tests/fuzzing/client2 tests/fuzzing/sync \ No newline at end of file diff --git a/CliClient/app/fuzzing.js b/CliClient/app/fuzzing.js new file mode 100644 index 000000000..414c0688c --- /dev/null +++ b/CliClient/app/fuzzing.js @@ -0,0 +1,208 @@ +require('source-map-support').install(); +require('babel-plugin-transform-runtime'); + +import { time } from 'lib/time-utils.js'; +import { Logger } from 'lib/logger.js'; +import lodash from 'lodash'; + +const exec = require('child_process').exec +const fs = require('fs-extra'); + +const baseDir = '/var/www/joplin/CliClient/tests/fuzzing'; +const syncDir = baseDir + '/sync'; +const joplinAppPath = __dirname + '/main.js'; + +const logger = new Logger(); +logger.addTarget('console'); +logger.setLevel(Logger.LEVEL_DEBUG); + +function createClient(id) { + return { + 'id': id, + 'profileDir': baseDir + '/client' + id, + }; +} + +async function createClients() { + let output = []; + let promises = []; + for (let clientId = 0; clientId < 2; clientId++) { + let client = createClient(clientId); + promises.push(fs.remove(client.profileDir)); + promises.push(execCommand(client, 'config sync.target local').then(() => { return execCommand(client, 'config sync.local.path ' + syncDir); })); + output.push(client); + } + + await Promise.all(promises); + + return output; +} + +function randomElement(array) { + if (!array.length) return null; + return array[Math.floor(Math.random() * array.length)]; +} + +function randomWord() { + const words = ['future','breezy','north','untidy','welcome','tenuous','material','tour','erect','bounce','skirt','compare','needle','abstracted','flower','detect','market','boring','lively','ragged','many','safe','credit','periodic','fold','whip','lewd','perform','nonchalant','rigid','amusing','giant','slippery','dog','tranquil','ajar','fanatical','flood','learned','helpless','size','ambiguous','long','six','jealous','history','distance','automatic','soggy','statuesque','prevent','full','price','parallel','mine','garrulous','wandering','puzzled','argument','sack','boil','marked','alive','observe','earsplitting','loving','fallacious','ice','parched','gleaming','horse','frame','gorgeous','quartz','quill','found','stranger','digestion','balance','cut','savory','peace','passenger','driving','sand','offer','rightful','earthquake','ear','spark','seashore','godly','rabbits','time','flowers','womanly','sulky','penitent','detail','warm','functional','silver','bushes','veil','filthy','jar','stitch','heartbreaking','bite-sized','station','play','plastic','common','save','subsequent','miscreant','slimy','train','disgusted','new','crib','boundless','stop','zephyr','roof','boiling','humdrum','record','park','symptomatic','vegetable','interest','ring','dusty','pet','depressed','murder','humor','capricious','kiss','gold','fax','cycle','river','black','four','irritating','mature','well-groomed','guard','hand','spotty','celery','air','scent','jelly','alleged','preach','anger','daffy','wrestle','torpid','excuse','jump','paint','exotic','tasty','auspicious','shirt','exercise','planes','romantic','telephone','teaching','towering','line','grouchy','eggnog','treat','powerful','abortive','paddle','belief','smash','fowl','steam','scale','workable','overwrought','elated','rustic','cuddly','star','extra-small','wacky','marry','optimal','muddle','care','turn','wealthy','phobic','ticket','petite','order','curly','lazy','careful','unequaled','mountain','attract','guide','robin','plant','hook','sail','creature','sparkle','sugar','volcano','grate','plough','undesirable','clever','mark','sea','responsible','destroy','broken','bore','spell','gate','lean','eye','afternoon','grease','note','smiling','puzzling','annoy','disagreeable','valuable','judge','frequent','live','gentle','reward','calm','aloof','old-fashioned','rule','sweet','hat','lumber','cheer','writing','able','roasted','scream','awful','meaty','nutty','trade','protest','letter','half','spiteful','library','food','sign','side','adhesive','itch','fuzzy','force','circle','historical','door','behavior','smile','bitter','scatter','crow','risk','rebel','milky','wise','rule','confuse','motion','roll','grain','structure','ship','admire','discreet','test','ask','meddle','tacit','abundant','skin','wound','beds','saw','few','rhyme','heavenly','jaded','finger','advice','letters','satisfying','general','add','fork','impartial','remind','rate','rotten','beam','puffy','march','horn','practise','brief','coordinated','ahead','woebegone','insidious','continue','rapid','adamant','gray','bless','dinosaurs','dress','woman','stir','songs','unwieldy','jump','cows','dust','terrify','acrid','illegal','desire','share','strange','damaged','entertaining','stare','underwear','legal','oven','refuse','accidental','blot','snakes','talk','lunchroom','man','blushing','waste','aggressive','oval','tax','clam','present','important','chicken','name','town','mend','knowing','long','wrathful','kettle','difficult','account','choke','decorate','bead','fear','majestic','shame','laborer','wine','story','hissing','stingy','plant','potato','houses','leg','number','condemned','hollow','bashful','distinct','ray','evanescent','whimsical','magic','bomb','cute','omniscient','plane','immense','brake','time','marvelous','mask','conscious','explain','answer','physical','berry','guide','machine','toad','business','milk','examine','chickens','uppity','red','kind','medical','shiver','punch','lake','sleepy','axiomatic','matter','nosy','zealous','mint','embarrassed','psychedelic','imagine','collar','tame','wing','soup','efficient','rat','signal','delight','belong','ducks','wicked','nod','close','snotty','measly','front','flag','smoke','magenta','squash','bubble','downtown','thirsty','tremendous','closed','stupid','shaggy','receipt','low','famous','momentous','grateful','concerned','tart','bomb','existence','vacation','grandfather','duck','bubble','reason','glue','assorted','peaceful','questionable','type','industry','chemical','rambunctious','plant','heap','church','suggestion','tickle','income','aberrant','enormous','knock']; + return randomElement(words); +} + +function execCommand(client, command) { + let exePath = 'node ' + joplinAppPath; + let cmd = exePath + ' --profile ' + client.profileDir + ' ' + command; + //logger.info(cmd.substr(exePath.length + 1)); + logger.info(cmd); + + return new Promise((resolve, reject) => { + exec(cmd, (error, stdout, stderr) => { + if (error) { + logger.error(stderr); + reject(error); + } else { + resolve(stdout); + } + }); + }); +} + +async function execRandomCommand(client) { + let possibleCommands = [ + ['mkbook {word}', 30], + ['mknote {word}', 100], + [async () => { + let items = await execCommand(client, 'dump'); + items = JSON.parse(items); + let item = randomElement(items); + if (!item) return; + + if (item.type_ == 1) { + await execCommand(client, 'rm -f ' + item.title); + } else if (item.type_ == 2) { + await execCommand(client, 'rm -f ' + '../' + item.title); + } else { + throw new Error('Unknown type: ' + item.type_); + } + }, 40], + ['sync', 10], + ]; + + let cmd = null; + while (true) { + cmd = randomElement(possibleCommands); + let r = 1 + Math.floor(Math.random() * 100); + if (r <= cmd[1]) break; + } + + cmd = cmd[0]; + + if (typeof cmd === 'function') { + return cmd(); + } else { + cmd = cmd.replace('{word}', randomWord()); + return execCommand(client, cmd); + } +} + +function randomNextCheckTime() { + let output = time.unixMs() + 1000 + Math.random() * 1000 * 10; + logger.info('Next sync check: ' + time.unixMsToIso(output) + ' (' + (Math.round((output - time.unixMs()) / 1000)) + ' sec.)'); + return output; +} + +async function compareClientItems(clientItems) { + let itemCounts = []; + for (let i = 0; i < clientItems.length; i++) { + let items = clientItems[i]; + itemCounts.push(items.length); + } + logger.info('Item count: ' + itemCounts.join(', ')); + + let r = lodash.uniq(itemCounts); + if (r.length > 1) { + logger.error('Item count is different'); + + await time.sleep(2); // Let the logger finish writing + process.exit(1); + } +} + +async function main(argv) { + await fs.remove(syncDir); + + let clients = await createClients(); + let activeCommandCounts = []; + let clientId = 0; + + for (let i = 0; i < clients.length; i++) { + clients[i].activeCommandCount = 0; + } + + function handleCommand(clientId) { + if (clients[clientId].activeCommandCount >= 1) return; + + clients[clientId].activeCommandCount++; + + execRandomCommand(clients[clientId]).catch((error) => { + logger.info('Client ' + clientId + ':'); + logger.error(error); + }).then((r) => { + if (r) { + logger.info('Client ' + clientId + ':'); + logger.info(r); + } + clients[clientId].activeCommandCount--; + }); + } + + let nextSyncCheckTime = randomNextCheckTime(); + let state = 'commands'; + + setInterval(async () => { + if (state == 'waitForSyncCheck') return; + + if (state == 'syncCheck') { + state = 'waitForSyncCheck'; + let clientItems = []; + // In order for all the clients to send their items and get those from the other + // clients, they need to perform 2 sync. + for (let loopCount = 0; loopCount < 2; loopCount++) { + for (let i = 0; i < clients.length; i++) { + await execCommand(clients[i], 'sync'); + if (loopCount === 1) { + let dump = await execCommand(clients[i], 'dump'); + clientItems[i] = JSON.parse(dump); + } + } + } + + await compareClientItems(clientItems); + + nextSyncCheckTime = randomNextCheckTime(); + state = 'commands'; + return; + } + + if (state == 'waitForClients') { + for (let i = 0; i < clients.length; i++) { + if (clients[i].activeCommandCount > 0) return; + } + + state = 'syncCheck'; + return; + } + + if (state == 'commands') { + if (nextSyncCheckTime <= time.unixMs()) { + state = 'waitForClients'; + return; + } + + handleCommand(clientId); + clientId++; + if (clientId >= clients.length) clientId = 0; + } + }, 100); +} + +main(process.argv); \ No newline at end of file diff --git a/CliClient/app/main.js b/CliClient/app/main.js index d60670fcf..d88441147 100644 --- a/CliClient/app/main.js +++ b/CliClient/app/main.js @@ -182,10 +182,14 @@ commands.push({ commands.push({ usage: 'rm ', description: 'Deletes the given item. For a notebook, all the notes within that notebook will be deleted. Use `rm ../` to delete a notebook.', + options: [ + ['-f, --force', 'Deletes the items without asking for confirmation.'], + ], action: async function(args, end) { try { let pattern = args['pattern']; let itemType = null; + let force = args.options && args.options.force === true; if (pattern.indexOf('*') < 0) { // Handle it as a simple title if (pattern.substr(0, 3) == '../') { @@ -197,16 +201,19 @@ commands.push({ let item = await BaseItem.loadItemByField(itemType, 'title', pattern); if (!item) throw new Error(_('No item with title "%s" found.', pattern)); - await BaseItem.deleteItem(itemType, item.id); - if (currentFolder && currentFolder.id == item.id) { - let f = await Folder.defaultFolder(); - switchCurrentFolder(f); + let ok = force ? true : await cmdPromptConfirm(this, _('Delete item?')); + if (ok) { + await BaseItem.deleteItem(itemType, item.id); + if (currentFolder && currentFolder.id == item.id) { + let f = await Folder.defaultFolder(); + switchCurrentFolder(f); + } } } else { // Handle it as a glob pattern let notes = await Note.previews(currentFolder.id, { titlePattern: pattern }); if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern)); - let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length)); + let ok = force ? true : await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length)); if (ok) { for (let i = 0; i < notes.length; i++) { await Note.delete(notes[i].id); @@ -246,6 +253,31 @@ commands.push({ autocomplete: autocompleteItems, }); +commands.push({ + usage: 'dump', + description: 'Dumps the complete database as JSON.', + action: async function(args, end) { + try { + let items = []; + let folders = await Folder.all(); + for (let i = 0; i < folders.length; i++) { + let folder = folders[i]; + let notes = await Note.previews(folder.id); + items.push(folder); + console.info(folder.title); + items = items.concat(notes); + } + + this.log(JSON.stringify(items)); + } catch (error) { + this.log(error); + } + + end(); + }, + autocomplete: autocompleteFolders, +}); + commands.push({ usage: 'ls [pattern]', description: 'Displays the notes in [notebook]. Use `ls ..` to display the list of notebooks.', @@ -254,6 +286,7 @@ commands.push({ ['-s, --sort ', 'Sorts the item by (eg. title, updated_time, created_time).'], ['-r, --reverse', 'Reverses the sorting order.'], ['-t, --type ', 'Displays only the items of the specific type(s). Can be `n` for notes, `t` for todos, or `nt` for notes and todos (eg. `-tt` would display only the todos, while `-ttd` would display notes and todos.'], + ['-f, --format ', 'Either "text" or "json"'], ], action: async function(args, end) { try { @@ -284,14 +317,18 @@ commands.push({ items = await Note.previews(currentFolder.id, queryOptions); } - for (let i = 0; i < items.length; i++) { - let item = items[i]; - let line = ''; - if (!!item.is_todo) { - line += sprintf('[%s] ', !!item.todo_completed ? 'X' : ' '); + if (options.format && options.format == 'json') { + this.log(JSON.stringify(items)); + } else { + for (let i = 0; i < items.length; i++) { + let item = items[i]; + let line = ''; + if (!!item.is_todo) { + line += sprintf('[%s] ', !!item.todo_completed ? 'X' : ' '); + } + line += item.title + suffix; + this.log(line); } - line += item.title + suffix; - this.log(line); } } catch (error) { this.log(error); @@ -774,6 +811,7 @@ async function main() { let cmd = shellArgsToString(argv); await vorpal.exec(cmd); await vorpal.exec('exit'); + await time.sleep(1); // Let loggers finish writing return; } else { vorpal.delimiter(promptString()).show(); diff --git a/CliClient/package.json b/CliClient/package.json index fa54e8560..9f5100afe 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -7,7 +7,7 @@ "url": "https://github.com/laurent22/joplin" }, "url": "git://github.com/laurent22/joplin.git", - "version": "0.8.21", + "version": "0.8.27", "bin": { "joplin": "./main.sh" }, diff --git a/CliClient/run_test.sh b/CliClient/run_test.sh index cc25fe110..f152edba1 100755 --- a/CliClient/run_test.sh +++ b/CliClient/run_test.sh @@ -7,5 +7,5 @@ ln -s "$CLIENT_DIR/build/lib" "$CLIENT_DIR/tests-build" npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js tests-build/base-model.js -#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js -#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/models/folder.js \ No newline at end of file +#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/synchronizer.js +#npm run build && NODE_PATH="$CLIENT_DIR/tests-build/" npm test tests-build/base-model.js \ No newline at end of file diff --git a/CliClient/tests/fuzzing/fuzzing.php b/CliClient/tests/fuzzing/fuzzing.php deleted file mode 100644 index c1233ae0b..000000000 --- a/CliClient/tests/fuzzing/fuzzing.php +++ /dev/null @@ -1,71 +0,0 @@ - $id, - 'profileDir' => dirname(__FILE__) . '/client' . $id, - ); -} - -$client = createClient($clientId); - -function deltree($dir) { - $files = array_diff(scandir($dir), array('.','..')); - foreach ($files as $file) { - is_dir("$dir/$file") ? deltree("$dir/$file") : unlink("$dir/$file"); - } - return rmdir($dir); -} - -function randomElement($array) { - return $array[mt_rand(0, count($array) - 1)]; -} - -function randomWord() { - $words = array('future','breezy','north','untidy','welcome','tenuous','material','tour','erect','bounce','skirt','compare','needle','abstracted','flower','detect','market','boring','lively','ragged','many','safe','credit','periodic','fold','whip','lewd','perform','nonchalant','rigid','amusing','giant','slippery','dog','tranquil','ajar','fanatical','flood','learned','helpless','size','ambiguous','long','six','jealous','history','distance','automatic','soggy','statuesque','prevent','full','price','parallel','mine','garrulous','wandering','puzzled','argument','sack','boil','marked','alive','observe','earsplitting','loving','fallacious','ice','parched','gleaming','horse','frame','gorgeous','quartz','quill','found','stranger','digestion','balance','cut','savory','peace','passenger','driving','sand','offer','rightful','earthquake','ear','spark','seashore','godly','rabbits','time','flowers','womanly','sulky','penitent','detail','warm','functional','silver','bushes','veil','filthy','jar','stitch','heartbreaking','bite-sized','station','play','plastic','common','save','subsequent','miscreant','slimy','train','disgusted','new','crib','boundless','stop','zephyr','roof','boiling','humdrum','record','park','symptomatic','vegetable','interest','ring','dusty','pet','depressed','murder','humor','capricious','kiss','gold','fax','cycle','river','black','four','irritating','mature','well-groomed','guard','hand','spotty','celery','air','scent','jelly','alleged','preach','anger','daffy','wrestle','torpid','excuse','jump','paint','exotic','tasty','auspicious','shirt','exercise','planes','romantic','telephone','teaching','towering','line','grouchy','eggnog','treat','powerful','abortive','paddle','belief','smash','fowl','steam','scale','workable','overwrought','elated','rustic','cuddly','star','extra-small','wacky','marry','optimal','muddle','care','turn','wealthy','phobic','ticket','petite','order','curly','lazy','careful','unequaled','mountain','attract','guide','robin','plant','hook','sail','creature','sparkle','sugar','volcano','grate','plough','undesirable','clever','mark','sea','responsible','destroy','broken','bore','spell','gate','lean','eye','afternoon','grease','note','smiling','puzzling','annoy','disagreeable','valuable','judge','frequent','live','gentle','reward','calm','aloof','old-fashioned','rule','sweet','hat','lumber','cheer','writing','able','roasted','scream','awful','meaty','nutty','trade','protest','letter','half','spiteful','library','food','sign','side','adhesive','itch','fuzzy','force','circle','historical','door','behavior','smile','bitter','scatter','crow','risk','rebel','milky','wise','rule','confuse','motion','roll','grain','structure','ship','admire','discreet','test','ask','meddle','tacit','abundant','skin','wound','beds','saw','few','rhyme','heavenly','jaded','finger','advice','letters','satisfying','general','add','fork','impartial','remind','rate','rotten','beam','puffy','march','horn','practise','brief','coordinated','ahead','woebegone','insidious','continue','rapid','adamant','gray','bless','dinosaurs','dress','woman','stir','songs','unwieldy','jump','cows','dust','terrify','acrid','illegal','desire','share','strange','damaged','entertaining','stare','underwear','legal','oven','refuse','accidental','blot','snakes','talk','lunchroom','man','blushing','waste','aggressive','oval','tax','clam','present','important','chicken','name','town','mend','knowing','long','wrathful','kettle','difficult','account','choke','decorate','bead','fear','majestic','shame','laborer','wine','story','hissing','stingy','plant','potato','houses','leg','number','condemned','hollow','bashful','distinct','ray','evanescent','whimsical','magic','bomb','cute','omniscient','plane','immense','brake','time','marvelous','mask','conscious','explain','answer','physical','berry','guide','machine','toad','business','milk','examine','chickens','uppity','red','kind','medical','shiver','punch','lake','sleepy','axiomatic','matter','nosy','zealous','mint','embarrassed','psychedelic','imagine','collar','tame','wing','soup','efficient','rat','signal','delight','belong','ducks','wicked','nod','close','snotty','measly','front','flag','smoke','magenta','squash','bubble','downtown','thirsty','tremendous','closed','stupid','shaggy','receipt','low','famous','momentous','grateful','concerned','tart','bomb','existence','vacation','grandfather','duck','bubble','reason','glue','assorted','peaceful','questionable','type','industry','chemical','rambunctious','plant','heap','church','suggestion','tickle','income','aberrant','enormous','knock'); - return randomElement($words); -} - -function execCommand($cmd) { - global $client; - $cmd = 'joplin --profile ' . $client['profileDir'] . ' ' . $cmd; - echo $cmd . "\n"; - exec($cmd, $output); - return implode("\n", $output); -} - -function execRandomCommand() { - $possibleCommands = array( - array('mkbook {word}', 10), - array('mknote {word}', 100), - array('sync', 10), - ); - - $cmd = null; - while (true) { - $cmd = randomElement($possibleCommands); - $r = mt_rand(1,100); - if ($r <= $cmd[1]) break; - } - - $cmd = $cmd[0]; - $cmd = str_replace('{word}', randomWord(), $cmd); - - return execCommand($cmd); -} - -deltree($client['profileDir']); - -execCommand('config sync.target local'); -execCommand('config sync.local.path ' . dirname(__FILE__) . '/sync'); - -while (true) { - $r = execRandomCommand(); - if ($r) { - echo $r . "\n"; - } -} \ No newline at end of file diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index 7dfe3a001..3ddccaa45 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -23,6 +23,7 @@ async function localItemsSameAsRemote(locals, expect) { try { let files = await fileApi().list(); files = files.items; + expect(locals.length).toBe(files.length); for (let i = 0; i < locals.length; i++) { @@ -31,6 +32,8 @@ async function localItemsSameAsRemote(locals, expect) { let remote = await fileApi().stat(path); expect(!!remote).toBe(true); + if (!remote) continue; + expect(remote.updated_time).toBe(dbItem.updated_time); let remoteContent = await fileApi().get(path); @@ -51,259 +54,248 @@ describe('Synchronizer', function() { done(); }); - it('should create remote items', async (done) => { - let folder = await Folder.save({ title: "folder1" }); - await Note.save({ title: "un", parent_id: folder.id }); + // it('should create remote items', async (done) => { + // let folder = await Folder.save({ title: "folder1" }); + // await Note.save({ title: "un", parent_id: folder.id }); - let all = await allItems(); + // let all = await allItems(); - await synchronizer().start(); + // await synchronizer().start(); - await localItemsSameAsRemote(all, expect); + // await localItemsSameAsRemote(all, expect); - done(); - }); + // done(); + // }); - it('should update remote item', async (done) => { - let folder = await Folder.save({ title: "folder1" }); - let note = await Note.save({ title: "un", parent_id: folder.id }); - await synchronizer().start(); + // it('should update remote item', async (done) => { + // let folder = await Folder.save({ title: "folder1" }); + // let note = await Note.save({ title: "un", parent_id: folder.id }); + // await synchronizer().start(); - await sleep(0.1); + // await sleep(0.1); - await Note.save({ title: "un UPDATE", id: note.id }); + // await Note.save({ title: "un UPDATE", id: note.id }); - let all = await allItems(); - await synchronizer().start(); + // let all = await allItems(); + // await synchronizer().start(); - await localItemsSameAsRemote(all, expect); + // await localItemsSameAsRemote(all, expect); - done(); - }); + // done(); + // }); - it('should create local items', async (done) => { - let folder = await Folder.save({ title: "folder1" }); - await Note.save({ title: "un", parent_id: folder.id }); - await synchronizer().start(); + // it('should create local items', async (done) => { + // let folder = await Folder.save({ title: "folder1" }); + // await Note.save({ title: "un", parent_id: folder.id }); + // await synchronizer().start(); - await switchClient(2); + // await switchClient(2); - await synchronizer().start(); + // await synchronizer().start(); - let all = await allItems(); - await localItemsSameAsRemote(all, expect); + // let all = await allItems(); + // await localItemsSameAsRemote(all, expect); - done(); - }); + // done(); + // }); - it('should update local items', async (done) => { - let folder1 = await Folder.save({ title: "folder1" }); - let note1 = await Note.save({ title: "un", parent_id: folder1.id }); - await synchronizer().start(); + // it('should update local items', async (done) => { + // let folder1 = await Folder.save({ title: "folder1" }); + // let note1 = await Note.save({ title: "un", parent_id: folder1.id }); + // await synchronizer().start(); - await switchClient(2); + // await switchClient(2); - await synchronizer().start(); + // await synchronizer().start(); - await sleep(0.1); + // await sleep(0.1); - let note2 = await Note.load(note1.id); - note2.title = "Updated on client 2"; - await Note.save(note2); - note2 = await Note.load(note2.id); + // let note2 = await Note.load(note1.id); + // note2.title = "Updated on client 2"; + // await Note.save(note2); + // note2 = await Note.load(note2.id); - await synchronizer().start(); + // await synchronizer().start(); - await switchClient(1); + // await switchClient(1); - await synchronizer().start(); + // await synchronizer().start(); - let all = await allItems(); + // let all = await allItems(); - await localItemsSameAsRemote(all, expect); + // await localItemsSameAsRemote(all, expect); - done(); - }); + // done(); + // }); - it('should resolve note conflicts', async (done) => { - let folder1 = await Folder.save({ title: "folder1" }); - let note1 = await Note.save({ title: "un", parent_id: folder1.id }); - await synchronizer().start(); + // it('should resolve note conflicts', async (done) => { + // let folder1 = await Folder.save({ title: "folder1" }); + // let note1 = await Note.save({ title: "un", parent_id: folder1.id }); + // await synchronizer().start(); - await switchClient(2); + // await switchClient(2); - await synchronizer().start(); + // await synchronizer().start(); - await sleep(0.1); + // await sleep(0.1); - let note2 = await Note.load(note1.id); - note2.title = "Updated on client 2"; - await Note.save(note2); - note2 = await Note.load(note2.id); + // let note2 = await Note.load(note1.id); + // note2.title = "Updated on client 2"; + // await Note.save(note2); + // note2 = await Note.load(note2.id); - await synchronizer().start(); + // await synchronizer().start(); - await switchClient(1); + // await switchClient(1); - await sleep(0.1); + // await sleep(0.1); - let note2conf = await Note.load(note1.id); - note2conf.title = "Updated on client 1"; - await Note.save(note2conf); - note2conf = await Note.load(note1.id); + // let note2conf = await Note.load(note1.id); + // note2conf.title = "Updated on client 1"; + // await Note.save(note2conf); + // note2conf = await Note.load(note1.id); - await synchronizer().start(); + // await synchronizer().start(); - let conflictedNotes = await Note.conflictedNotes(); + // let conflictedNotes = await Note.conflictedNotes(); - expect(conflictedNotes.length).toBe(1); + // expect(conflictedNotes.length).toBe(1); - // Other than the id (since the conflicted note is a duplicate), and the is_conflict property - // the conflicted and original note must be the same in every way, to make sure no data has been lost. - let conflictedNote = conflictedNotes[0]; - expect(conflictedNote.id == note2conf.id).toBe(false); - for (let n in conflictedNote) { - if (!conflictedNote.hasOwnProperty(n)) continue; - if (n == 'id' || n == 'is_conflict') continue; - expect(conflictedNote[n]).toBe(note2conf[n], 'Property: ' + n); - } + // // Other than the id (since the conflicted note is a duplicate), and the is_conflict property + // // the conflicted and original note must be the same in every way, to make sure no data has been lost. + // let conflictedNote = conflictedNotes[0]; + // expect(conflictedNote.id == note2conf.id).toBe(false); + // for (let n in conflictedNote) { + // if (!conflictedNote.hasOwnProperty(n)) continue; + // if (n == 'id' || n == 'is_conflict') continue; + // expect(conflictedNote[n]).toBe(note2conf[n], 'Property: ' + n); + // } - let noteUpdatedFromRemote = await Note.load(note1.id); - for (let n in noteUpdatedFromRemote) { - if (!noteUpdatedFromRemote.hasOwnProperty(n)) continue; - if (n == 'sync_time') continue; - expect(noteUpdatedFromRemote[n]).toBe(note2[n], 'Property: ' + n); - } + // let noteUpdatedFromRemote = await Note.load(note1.id); + // for (let n in noteUpdatedFromRemote) { + // if (!noteUpdatedFromRemote.hasOwnProperty(n)) continue; + // if (n == 'sync_time') continue; + // expect(noteUpdatedFromRemote[n]).toBe(note2[n], 'Property: ' + n); + // } - done(); - }); + // done(); + // }); - it('should resolve folders conflicts', async (done) => { - let folder1 = await Folder.save({ title: "folder1" }); - let note1 = await Note.save({ title: "un", parent_id: folder1.id }); - await synchronizer().start(); + // it('should resolve folders conflicts', async (done) => { + // let folder1 = await Folder.save({ title: "folder1" }); + // let note1 = await Note.save({ title: "un", parent_id: folder1.id }); + // await synchronizer().start(); - await switchClient(2); // ---------------------------------- + // await switchClient(2); // ---------------------------------- - await synchronizer().start(); + // await synchronizer().start(); - await sleep(0.1); + // await sleep(0.1); - let folder1_modRemote = await Folder.load(folder1.id); - folder1_modRemote.title = "folder1 UPDATE CLIENT 2"; - await Folder.save(folder1_modRemote); - folder1_modRemote = await Folder.load(folder1_modRemote.id); + // let folder1_modRemote = await Folder.load(folder1.id); + // folder1_modRemote.title = "folder1 UPDATE CLIENT 2"; + // await Folder.save(folder1_modRemote); + // folder1_modRemote = await Folder.load(folder1_modRemote.id); - await synchronizer().start(); + // await synchronizer().start(); - await switchClient(1); // ---------------------------------- + // await switchClient(1); // ---------------------------------- - await sleep(0.1); + // await sleep(0.1); - let folder1_modLocal = await Folder.load(folder1.id); - folder1_modLocal.title = "folder1 UPDATE CLIENT 1"; - await Folder.save(folder1_modLocal); - folder1_modLocal = await Folder.load(folder1.id); + // let folder1_modLocal = await Folder.load(folder1.id); + // folder1_modLocal.title = "folder1 UPDATE CLIENT 1"; + // await Folder.save(folder1_modLocal); + // folder1_modLocal = await Folder.load(folder1.id); - await synchronizer().start(); + // await synchronizer().start(); - let folder1_final = await Folder.load(folder1.id); - expect(folder1_final.title).toBe(folder1_modRemote.title); + // let folder1_final = await Folder.load(folder1.id); + // expect(folder1_final.title).toBe(folder1_modRemote.title); - done(); - }); + // done(); + // }); - it('should delete remote items', async (done) => { - let folder1 = await Folder.save({ title: "folder1" }); - let note1 = await Note.save({ title: "un", parent_id: folder1.id }); - await synchronizer().start(); + // it('should delete remote notes', async (done) => { + // let folder1 = await Folder.save({ title: "folder1" }); + // let note1 = await Note.save({ title: "un", parent_id: folder1.id }); + // await synchronizer().start(); - await switchClient(2); + // await switchClient(2); - await synchronizer().start(); + // await synchronizer().start(); - await sleep(0.1); + // await sleep(0.1); - await Note.delete(note1.id); + // await Note.delete(note1.id); - await synchronizer().start(); + // await synchronizer().start(); - let files = await fileApi().list(); - files = files.items; + // let files = await fileApi().list(); + // files = files.items; - expect(files.length).toBe(1); - expect(files[0].path).toBe(Folder.systemPath(folder1)); + // expect(files.length).toBe(1); + // expect(files[0].path).toBe(Folder.systemPath(folder1)); - let deletedItems = await BaseModel.deletedItems(); - expect(deletedItems.length).toBe(0); + // let deletedItems = await BaseModel.deletedItems(); + // expect(deletedItems.length).toBe(0); - done(); - }); + // done(); + // }); - it('should delete local items', async (done) => { - let folder1 = await Folder.save({ title: "folder1" }); - let note1 = await Note.save({ title: "un", parent_id: folder1.id }); - await synchronizer().start(); + // it('should delete local notes', async (done) => { + // let folder1 = await Folder.save({ title: "folder1" }); + // let note1 = await Note.save({ title: "un", parent_id: folder1.id }); + // await synchronizer().start(); - await switchClient(2); + // await switchClient(2); - await synchronizer().start(); + // await synchronizer().start(); - await sleep(0.1); + // await sleep(0.1); - await Note.delete(note1.id); + // await Note.delete(note1.id); - await synchronizer().start(); + // await synchronizer().start(); - await switchClient(1); + // await switchClient(1); - await synchronizer().start(); + // await synchronizer().start(); - let items = await allItems(); + // let items = await allItems(); - expect(items.length).toBe(1); + // expect(items.length).toBe(1); - let deletedItems = await BaseModel.deletedItems(); + // let deletedItems = await BaseModel.deletedItems(); - expect(deletedItems.length).toBe(0); + // expect(deletedItems.length).toBe(0); - done(); - }); + // done(); + // }); - it('should handle conflict when remote note is deleted then local note is modified', async (done) => { - let folder1 = await Folder.save({ title: "folder1" }); - let note1 = await Note.save({ title: "un", parent_id: folder1.id }); - await synchronizer().start(); + // it('should delete remote folder', async (done) => { + // let folder1 = await Folder.save({ title: "folder1" }); + // let folder2 = await Folder.save({ title: "folder2" }); + // await synchronizer().start(); - await switchClient(2); + // await switchClient(2); - await synchronizer().start(); + // await synchronizer().start(); - await sleep(0.1); + // await sleep(0.1); - await Note.delete(note1.id); + // await Folder.delete(folder2.id); - await synchronizer().start(); + // await synchronizer().start(); - await switchClient(1); - - let newTitle = 'Modified after having been deleted'; - await Note.save({ id: note1.id, title: newTitle }); - - await synchronizer().start(); - - let conflictedNotes = await Note.conflictedNotes(); - - expect(conflictedNotes.length).toBe(1); - expect(conflictedNotes[0].title).toBe(newTitle); + // localItemsSameAsRemote(); - done(); - }); + // done(); + // }); - it('should handle conflict when remote folder is deleted then local folder is renamed', async (done) => { + it('should delete local folder', async (done) => { let folder1 = await Folder.save({ title: "folder1" }); let folder2 = await Folder.save({ title: "folder2" }); - let note1 = await Note.save({ title: "un", parent_id: folder1.id }); await synchronizer().start(); await switchClient(2); @@ -312,60 +304,116 @@ describe('Synchronizer', function() { await sleep(0.1); - await Folder.delete(folder1.id); + await Folder.delete(folder2.id); await synchronizer().start(); await switchClient(1); - await sleep(0.1); - - let newTitle = 'Modified after having been deleted'; - await Folder.save({ id: folder1.id, title: newTitle }); - await synchronizer().start(); let items = await allItems(); - - expect(items.length).toBe(1); + localItemsSameAsRemote(items, expect); done(); }); - it('should allow duplicate folder title and rename the new one', async (done) => { - let localF1 = await Folder.save({ title: "folder" }); + // it('should handle conflict when remote note is deleted then local note is modified', async (done) => { + // let folder1 = await Folder.save({ title: "folder1" }); + // let note1 = await Note.save({ title: "un", parent_id: folder1.id }); + // await synchronizer().start(); - await switchClient(2); + // await switchClient(2); - let remoteF2 = await Folder.save({ title: "folder" }); - await synchronizer().start(); + // await synchronizer().start(); - await switchClient(1); + // await sleep(0.1); - await sleep(0.1); + // await Note.delete(note1.id); - await synchronizer().start(); + // await synchronizer().start(); - let localF2 = await Folder.load(remoteF2.id); + // await switchClient(1); - expect(localF2.title == remoteF2.title).toBe(false); + // let newTitle = 'Modified after having been deleted'; + // await Note.save({ id: note1.id, title: newTitle }); - // Then that folder that has been renamed locally should be set in such a way - // that synchronizing it applies the title change remotely, and that new title - // should be retrieved by client 2. + // await synchronizer().start(); - await synchronizer().start(); + // let conflictedNotes = await Note.conflictedNotes(); - await switchClient(2); - await sleep(0.1); + // expect(conflictedNotes.length).toBe(1); + // expect(conflictedNotes[0].title).toBe(newTitle); + + // done(); + // }); - await synchronizer().start(); + // it('should handle conflict when remote folder is deleted then local folder is renamed', async (done) => { + // let folder1 = await Folder.save({ title: "folder1" }); + // let folder2 = await Folder.save({ title: "folder2" }); + // let note1 = await Note.save({ title: "un", parent_id: folder1.id }); + // await synchronizer().start(); - remoteF2 = await Folder.load(remoteF2.id); + // await switchClient(2); - expect(remoteF2.title == localF2.title).toBe(true); + // await synchronizer().start(); - done(); - }); + // await sleep(0.1); + + // await Folder.delete(folder1.id); + + // await synchronizer().start(); + + // await switchClient(1); + + // await sleep(0.1); + + // let newTitle = 'Modified after having been deleted'; + // await Folder.save({ id: folder1.id, title: newTitle }); + + // await synchronizer().start(); + + // let items = await allItems(); + + // expect(items.length).toBe(1); + + // done(); + // }); + + // it('should allow duplicate folder title and rename the new one', async (done) => { + // let localF1 = await Folder.save({ title: "folder" }); + + // await switchClient(2); + + // let remoteF2 = await Folder.save({ title: "folder" }); + // await synchronizer().start(); + + // await switchClient(1); + + // await sleep(0.1); + + // await synchronizer().start(); + + // let localF2 = await Folder.load(remoteF2.id); + + // expect(localF2.title == remoteF2.title).toBe(false); + + // // Then that folder that has been renamed locally should be set in such a way + // // that synchronizing it applies the title change remotely, and that new title + // // should be retrieved by client 2. + + // await synchronizer().start(); + + // await switchClient(2); + // await sleep(0.1); + + // await synchronizer().start(); + + // remoteF2 = await Folder.load(remoteF2.id); + + // expect(remoteF2.title == localF2.title).toBe(true); + + // done(); + // }); }); \ No newline at end of file diff --git a/lib/logger.js b/lib/logger.js index caebb3070..9e383294a 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -66,32 +66,34 @@ class Logger { serializedObject = object; } - this.fileAppendQueue_.push({ - path: t.path, - line: line + serializedObject + "\n", - }); + fs.appendFileSync(t.path, line + serializedObject + "\n"); - this.scheduleFileAppendQueueProcessing_(); + // this.fileAppendQueue_.push({ + // path: t.path, + // line: line + serializedObject + "\n", + // }); + + // this.scheduleFileAppendQueueProcessing_(); } else if (t.type == 'vorpal') { t.vorpal.log(object); } } } - scheduleFileAppendQueueProcessing_() { - if (this.fileAppendQueueTID_) return; + // scheduleFileAppendQueueProcessing_() { + // if (this.fileAppendQueueTID_) return; - this.fileAppendQueueTID_ = setTimeout(async () => { - this.fileAppendQueueTID_ = null; + // this.fileAppendQueueTID_ = setTimeout(async () => { + // this.fileAppendQueueTID_ = null; - let queue = this.fileAppendQueue_.slice(0); - for (let i = 0; i < queue.length; i++) { - let t = queue[i]; - await fs.appendFile(t.path, t.line); - } - this.fileAppendQueue_.splice(0, queue.length); - }, 10); - } + // let queue = this.fileAppendQueue_.slice(0); + // for (let i = 0; i < queue.length; i++) { + // let t = queue[i]; + // await fs.appendFile(t.path, t.line); + // } + // this.fileAppendQueue_.splice(0, queue.length); + // }, 1); + // } error(object) { return this.log(Logger.LEVEL_ERROR, object); } warn(object) { return this.log(Logger.LEVEL_WARN, object); } diff --git a/lib/models/folder.js b/lib/models/folder.js index 798334166..a78785b6f 100644 --- a/lib/models/folder.js +++ b/lib/models/folder.js @@ -104,7 +104,11 @@ class Folder extends BaseItem { static save(o, options = null) { return Folder.loadByField('title', o.title).then((existingFolder) => { - if (existingFolder && existingFolder.id != o.id) throw new Error(_('A notebook with title "%s" already exists', o.title)); + if (existingFolder && existingFolder.id != o.id) { + let error = new Error(_('A notebook with title "%s" already exists', o.title)); + error.code = 'duplicateTitle'; + throw error; + } return super.save(o, options).then((folder) => { this.dispatch({ diff --git a/lib/synchronizer.js b/lib/synchronizer.js index 3672747f9..c060888b8 100644 --- a/lib/synchronizer.js +++ b/lib/synchronizer.js @@ -63,6 +63,7 @@ class Synchronizer { async logSyncSummary(report) { for (let n in report) { + if (!report.hasOwnProperty(n)) continue; this.logger().info(n + ': ' + (report[n] ? report[n] : '-')); } let folderCount = await Folder.count(); @@ -262,7 +263,7 @@ class Synchronizer { } else { if (remote.updated_time > local.updated_time) { action = 'updateLocal'; - reason = sprintf('remote is more recent than local'); // , time.unixMsToIso(remote.updated_time), time.unixMsToIso(local.updated_time) + reason = sprintf('remote is more recent than local'); } } @@ -284,7 +285,18 @@ class Synchronizer { newContent.sync_time = time.unixMs(); let options = { autoTimestamp: false }; if (action == 'createLocal') options.isNew = true; - await ItemClass.save(newContent, options); + try { + await ItemClass.save(newContent, options); + } catch (error) { + + if (error.code == 'duplicateTitle') { + newContent.title = newContent.title + '-' + newContent.created_time + '-' + (Math.floor(Math.random() * 1000)); + newContent.updated_time = newContent.sync_time + 2; + await ItemClass.save(newContent, options); + } else { + throw error; + } + } this.logSyncOperation(action, local, content, reason); } else {