From 08af9de19006678eb55e26c7d6935e6e28172300 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Mon, 6 May 2019 21:35:29 +0100 Subject: [PATCH] All: Resolves #712: New: Support for note history (#1415) * Started revisions support * More rev changes * More rev changes * More revs changes * Fixed deletion algorithm * More tests and moved updated time to separate field * Display info when restoring note * Better handling of existing notes * wip * Further improvements and fixed tests * Better handling of changes created via sync * Enable chokidar again * Testing special case * Further improved logic to handle notes that existed before the revision service * Added tests * Better handling of encrypted revisions * Improved handling of deleted note revisions by moving logic to collectRevision * Improved handling of old notes by moving logic to collectRevision() * Handle case when deleting revisions while one is still encrypted * UI tweaks * Added revision service to mobile app * Fixed config screens on mobile and desktop * Enabled revisions on CLI app --- CliClient/app/app.js | 3 + CliClient/app/main.js | 2 + CliClient/package-lock.json | 7 +- CliClient/package.json | 1 + CliClient/run_test.sh | 2 + CliClient/tests/encryption.js | 3 +- CliClient/tests/models_ItemChange.js | 10 +- CliClient/tests/models_Revision.js | 71 ++++ CliClient/tests/services_Revision.js | 372 ++++++++++++++++++ CliClient/tests/synchronizer.js | 234 ++++++++--- CliClient/tests/test-utils.js | 47 ++- ElectronClient/app/ElectronAppWrapper.js | 2 +- ElectronClient/app/app.js | 6 + ElectronClient/app/gui/ConfigScreen.jsx | 5 +- ElectronClient/app/gui/HelpButton.jsx | 39 ++ ElectronClient/app/gui/MainScreen.jsx | 2 + .../app/gui/NotePropertiesDialog.jsx | 34 +- ElectronClient/app/gui/NoteRevisionViewer.jsx | 175 ++++++++ ElectronClient/app/gui/NoteText.jsx | 28 +- ElectronClient/app/gui/NoteTextViewer.jsx | 14 +- ElectronClient/app/gui/SideBar.jsx | 2 +- ElectronClient/app/main-html.js | 2 + ElectronClient/app/package-lock.json | 264 +++++++++---- ElectronClient/app/package.json | 2 + ElectronClient/app/plugins/GotoAnything.jsx | 5 +- ElectronClient/app/style.css | 9 + ElectronClient/app/theme.js | 11 +- ReactNativeClient/lib/BaseApplication.js | 3 + ReactNativeClient/lib/BaseModel.js | 1 + .../lib/components/screens/config.js | 8 +- ReactNativeClient/lib/joplin-database.js | 31 +- ReactNativeClient/lib/models/BaseItem.js | 26 +- ReactNativeClient/lib/models/ItemChange.js | 10 +- ReactNativeClient/lib/models/Note.js | 49 ++- ReactNativeClient/lib/models/Revision.js | 248 ++++++++++++ ReactNativeClient/lib/models/Setting.js | 8 + .../lib/services/ExternalEditWatcher.js | 1 + .../lib/services/ItemChangeUtils.js | 1 + .../lib/services/ResourceService.js | 1 + .../lib/services/RevisionService.js | 277 +++++++++++++ ReactNativeClient/lib/synchronizer.js | 38 +- ReactNativeClient/package-lock.json | 52 ++- ReactNativeClient/package.json | 1 + ReactNativeClient/root.js | 9 + 44 files changed, 1873 insertions(+), 243 deletions(-) create mode 100644 CliClient/tests/models_Revision.js create mode 100644 CliClient/tests/services_Revision.js create mode 100644 ElectronClient/app/gui/HelpButton.jsx create mode 100644 ElectronClient/app/gui/NoteRevisionViewer.jsx create mode 100644 ReactNativeClient/lib/models/Revision.js create mode 100644 ReactNativeClient/lib/services/RevisionService.js diff --git a/CliClient/app/app.js b/CliClient/app/app.js index b660a12f1..fc975360a 100644 --- a/CliClient/app/app.js +++ b/CliClient/app/app.js @@ -23,6 +23,7 @@ const fs = require('fs-extra'); const { cliUtils } = require('./cli-utils.js'); const Cache = require('lib/Cache'); const WelcomeUtils = require('lib/WelcomeUtils'); +const RevisionService = require('lib/services/RevisionService'); class Application extends BaseApplication { @@ -422,6 +423,8 @@ class Application extends BaseApplication { const tags = await Tag.allWithNotes(); ResourceService.runInBackground(); + + RevisionService.instance().runInBackground(); this.dispatch({ type: 'TAG_UPDATE_ALL', diff --git a/CliClient/app/main.js b/CliClient/app/main.js index f6408bdd2..7ee9c1784 100644 --- a/CliClient/app/main.js +++ b/CliClient/app/main.js @@ -22,6 +22,7 @@ const Tag = require('lib/models/Tag.js'); const NoteTag = require('lib/models/NoteTag.js'); const MasterKey = require('lib/models/MasterKey'); const Setting = require('lib/models/Setting.js'); +const Revision = require('lib/models/Revision.js'); const { Logger } = require('lib/logger.js'); const { FsDriverNode } = require('lib/fs-driver-node.js'); const { shimInit } = require('lib/shim-init-node.js'); @@ -43,6 +44,7 @@ BaseItem.loadClass('Resource', Resource); BaseItem.loadClass('Tag', Tag); BaseItem.loadClass('NoteTag', NoteTag); BaseItem.loadClass('MasterKey', MasterKey); +BaseItem.loadClass('Revision', Revision); Setting.setConstant('appId', 'net.cozic.joplin-cli'); Setting.setConstant('appType', 'cli'); diff --git a/CliClient/package-lock.json b/CliClient/package-lock.json index dd60ded07..b023d944c 100644 --- a/CliClient/package-lock.json +++ b/CliClient/package-lock.json @@ -210,7 +210,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -560,6 +560,11 @@ "resolved": "https://registry.npmjs.org/diacritics/-/diacritics-1.3.0.tgz", "integrity": "sha1-PvqHMj67hj5mls67AILUj/PW96E=" }, + "diff-match-patch": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.4.tgz", + "integrity": "sha512-Uv3SW8bmH9nAtHKaKSanOQmj2DnlH65fUpcrMdfdaOxUG02QQ4YGZ8AE7kKOMisF7UqvOlGKVYWRvezdncW9lg==" + }, "domexception": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", diff --git a/CliClient/package.json b/CliClient/package.json index 98dd5d988..209642bf1 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -33,6 +33,7 @@ "base-64": "^0.1.0", "compare-version": "^0.1.2", "diacritics": "^1.3.0", + "diff-match-patch": "^1.0.4", "es6-promise-pool": "^2.5.0", "follow-redirects": "^1.2.4", "form-data": "^2.1.4", diff --git a/CliClient/run_test.sh b/CliClient/run_test.sh index 5d2458261..514e67215 100755 --- a/CliClient/run_test.sh +++ b/CliClient/run_test.sh @@ -31,6 +31,7 @@ npm test tests-build/models_Folder.js npm test tests-build/models_ItemChange.js npm test tests-build/models_Note.js npm test tests-build/models_Resource.js +npm test tests-build/models_Revision.js npm test tests-build/models_Setting.js npm test tests-build/models_Tag.js npm test tests-build/pathUtils.js @@ -38,6 +39,7 @@ npm test tests-build/services_InteropService.js npm test tests-build/services_ResourceService.js npm test tests-build/services_rest_Api.js npm test tests-build/services_SearchEngine.js +npm test tests-build/services_Revision.js npm test tests-build/StringUtils.js npm test tests-build/synchronizer.js npm test tests-build/urlUtils.js \ No newline at end of file diff --git a/CliClient/tests/encryption.js b/CliClient/tests/encryption.js index 63ca4a417..1603abdbb 100644 --- a/CliClient/tests/encryption.js +++ b/CliClient/tests/encryption.js @@ -25,8 +25,7 @@ describe('Encryption', function() { beforeEach(async (done) => { await setupDatabaseAndSynchronizer(1); - //await setupDatabaseAndSynchronizer(2); - //await switchClient(1); + await switchClient(1); service = new EncryptionService(); BaseItem.encryptionService_ = service; Setting.setValue('encryption.enabled', true); diff --git a/CliClient/tests/models_ItemChange.js b/CliClient/tests/models_ItemChange.js index 8e082931e..fbcf1b72a 100644 --- a/CliClient/tests/models_ItemChange.js +++ b/CliClient/tests/models_ItemChange.js @@ -1,7 +1,7 @@ require('app-module-path').addPath(__dirname); const { time } = require('lib/time-utils.js'); -const { asyncTest, fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js'); +const { asyncTest, fileContentEqual, revisionService, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js'); const SearchEngine = require('lib/services/SearchEngine'); const ResourceService = require('lib/services/ResourceService'); const ItemChangeUtils = require('lib/services/ItemChangeUtils'); @@ -34,19 +34,17 @@ describe('models_ItemChange', function() { const resourceService = new ResourceService(); await searchEngine.syncTables(); - // If we run this now, it should not delete any change because // the resource service has not yet processed the change await ItemChangeUtils.deleteProcessedChanges(); - expect(await ItemChange.lastChangeId()).toBe(1); await resourceService.indexNoteResources(); - - // Now that the resource service has processed the change, - // the change can be deleted. await ItemChangeUtils.deleteProcessedChanges(); + expect(await ItemChange.lastChangeId()).toBe(1); + await revisionService().collectRevisions(); + await ItemChangeUtils.deleteProcessedChanges(); expect(await ItemChange.lastChangeId()).toBe(0); })); diff --git a/CliClient/tests/models_Revision.js b/CliClient/tests/models_Revision.js new file mode 100644 index 000000000..82dff39b2 --- /dev/null +++ b/CliClient/tests/models_Revision.js @@ -0,0 +1,71 @@ +require('app-module-path').addPath(__dirname); + +const { time } = require('lib/time-utils.js'); +const { asyncTest, fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js'); +const Folder = require('lib/models/Folder.js'); +const Note = require('lib/models/Note.js'); +const NoteTag = require('lib/models/NoteTag.js'); +const Tag = require('lib/models/Tag.js'); +const Revision = require('lib/models/Revision.js'); +const BaseModel = require('lib/BaseModel.js'); +const { shim } = require('lib/shim'); + +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); +}); + +describe('models_Revision', function() { + + beforeEach(async (done) => { + await setupDatabaseAndSynchronizer(1); + await switchClient(1); + done(); + }); + + it('should create patches of text and apply it', asyncTest(async () => { + const note1 = await Note.save({ body: 'my note\nsecond line' }); + + const patch = Revision.createTextPatch(note1.body, 'my new note\nsecond line'); + const merged = Revision.applyTextPatch(note1.body, patch); + + expect(merged).toBe('my new note\nsecond line'); + })); + + it('should create patches of objects and apply it', asyncTest(async () => { + const oldObject = { + one: '123', + two: '456', + three: '789', + }; + + const newObject = { + one: '123', + three: '999', + } + + const patch = Revision.createObjectPatch(oldObject, newObject); + const merged = Revision.applyObjectPatch(oldObject, patch); + + expect(JSON.stringify(merged)).toBe(JSON.stringify(newObject)); + })); + + it('should move target revision to the top', asyncTest(async () => { + const revs = [ + { id: '123' }, + { id: '456' }, + { id: '789' }, + ]; + + let newRevs; + newRevs = Revision.moveRevisionToTop({ id: '456' }, revs); + expect(newRevs[0].id).toBe('123'); + expect(newRevs[1].id).toBe('789'); + expect(newRevs[2].id).toBe('456'); + + newRevs = Revision.moveRevisionToTop({ id: '789' }, revs); + expect(newRevs[0].id).toBe('123'); + expect(newRevs[1].id).toBe('456'); + expect(newRevs[2].id).toBe('789'); + })); + +}); \ No newline at end of file diff --git a/CliClient/tests/services_Revision.js b/CliClient/tests/services_Revision.js new file mode 100644 index 000000000..a6dcc5565 --- /dev/null +++ b/CliClient/tests/services_Revision.js @@ -0,0 +1,372 @@ +require('app-module-path').addPath(__dirname); + +const { time } = require('lib/time-utils.js'); +const { asyncTest, fileContentEqual, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js'); +const Folder = require('lib/models/Folder.js'); +const Setting = require('lib/models/Setting.js'); +const Note = require('lib/models/Note.js'); +const NoteTag = require('lib/models/NoteTag.js'); +const ItemChange = require('lib/models/ItemChange.js'); +const Tag = require('lib/models/Tag.js'); +const Revision = require('lib/models/Revision.js'); +const BaseModel = require('lib/BaseModel.js'); +const RevisionService = require('lib/services/RevisionService.js'); +const { shim } = require('lib/shim'); + +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); +}); + +describe('services_Revision', function() { + + beforeEach(async (done) => { + await setupDatabaseAndSynchronizer(1); + await switchClient(1); + done(); + }); + + it('should create diff and rebuild notes', asyncTest(async () => { + const service = new RevisionService(); + + const n1_v1 = await Note.save({ title: '', author: 'testing' }); + await service.collectRevisions(); + await Note.save({ id: n1_v1.id, title: 'hello', author: 'testing' }); + await service.collectRevisions(); + const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome', author: '' }); + await service.collectRevisions(); + + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id); + expect(revisions.length).toBe(2); + expect(revisions[1].parent_id).toBe(revisions[0].id); + + const rev1 = await service.revisionNote(revisions, 0); + expect(rev1.title).toBe('hello'); + expect(rev1.author).toBe('testing'); + + const rev2 = await service.revisionNote(revisions, 1); + expect(rev2.title).toBe('hello welcome'); + expect(rev2.author).toBe(''); + + await time.sleep(0.5); + + await service.deleteOldRevisions(400); + const revisions2 = await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id); + expect(revisions2.length).toBe(0); + })); + + it('should delete old revisions (1 note, 2 rev)', asyncTest(async () => { + const service = new RevisionService(); + + const n1_v0 = await Note.save({ title: '' }); + const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' }); + await service.collectRevisions(); + await time.sleep(1); + const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' }); + await service.collectRevisions(); + expect((await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id)).length).toBe(2); + + await service.deleteOldRevisions(1000); + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id); + expect(revisions.length).toBe(1); + + const rev1 = await service.revisionNote(revisions, 0); + expect(rev1.title).toBe('hello welcome'); + })); + + it('should delete old revisions (1 note, 3 rev)', asyncTest(async () => { + const service = new RevisionService(); + + const n1_v0 = await Note.save({ title: '' }); + const n1_v1 = await Note.save({ id: n1_v0.id, title: 'one' }); + await service.collectRevisions(); + await time.sleep(1); + const n1_v2 = await Note.save({ id: n1_v1.id, title: 'one two' }); + await service.collectRevisions(); + await time.sleep(1); + const n1_v3 = await Note.save({ id: n1_v1.id, title: 'one two three' }); + await service.collectRevisions(); + + { + await service.deleteOldRevisions(2000); + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id); + expect(revisions.length).toBe(2); + + const rev1 = await service.revisionNote(revisions, 0); + expect(rev1.title).toBe('one two'); + + const rev2 = await service.revisionNote(revisions, 1); + expect(rev2.title).toBe('one two three'); + } + + { + await service.deleteOldRevisions(1000); + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id); + expect(revisions.length).toBe(1); + + const rev1 = await service.revisionNote(revisions, 0); + expect(rev1.title).toBe('one two three'); + } + })); + + it('should delete old revisions (2 notes, 2 rev)', asyncTest(async () => { + const service = new RevisionService(); + + const n1_v0 = await Note.save({ title: '' }); + const n1_v1 = await Note.save({ id: n1_v0.id, title: 'note 1' }); + const n2_v0 = await Note.save({ title: '' }); + const n2_v1 = await Note.save({ id: n2_v0.id, title: 'note 2' }); + await service.collectRevisions(); + await time.sleep(1); + const n1_v2 = await Note.save({ id: n1_v1.id, title: 'note 1 (v2)' }); + const n2_v2 = await Note.save({ id: n2_v1.id, title: 'note 2 (v2)' }); + await service.collectRevisions(); + + expect((await Revision.all()).length).toBe(4); + + await service.deleteOldRevisions(1000); + + { + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id); + expect(revisions.length).toBe(1); + const rev1 = await service.revisionNote(revisions, 0); + expect(rev1.title).toBe('note 1 (v2)'); + } + + { + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n2_v1.id); + expect(revisions.length).toBe(1); + const rev1 = await service.revisionNote(revisions, 0); + expect(rev1.title).toBe('note 2 (v2)'); + } + })); + + it('should handle conflicts', asyncTest(async () => { + const service = new RevisionService(); + + // A conflict happens in this case: + // - Device 1 creates note1 (rev1) + // - Device 2 syncs and get note1 + // - Device 1 modifies note1 (rev2) + // - Device 2 modifies note1 (rev3) + // When reconstructing the notes based on the revisions, we need to make sure it follow the right + // "path". For example, to reconstruct the note at rev2 it would be: + // rev1 => rev2 + // To reconstruct the note at rev3 it would be: + // rev1 => rev3 + // And not, for example, rev1 => rev2 => rev3 + + const n1_v1 = await Note.save({ title: 'hello' }); + const noteId = n1_v1.id; + const rev1 = await service.createNoteRevision(n1_v1); + const n1_v2 = await Note.save({ id: noteId, title: 'hello Paul' }); + const rev2 = await service.createNoteRevision(n1_v2, rev1.id); + const n1_v3 = await Note.save({ id: noteId, title: 'hello John' }); + const rev3 = await service.createNoteRevision(n1_v3, rev1.id); + + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, noteId); + expect(revisions.length).toBe(3); + expect(revisions[1].parent_id).toBe(rev1.id); + expect(revisions[2].parent_id).toBe(rev1.id); + + const revNote1 = await service.revisionNote(revisions, 0); + const revNote2 = await service.revisionNote(revisions, 1); + const revNote3 = await service.revisionNote(revisions, 2); + expect(revNote1.title).toBe('hello'); + expect(revNote2.title).toBe('hello Paul'); + expect(revNote3.title).toBe('hello John'); + })); + + it('should create a revision for notes that existed before the revision service, the first time it is saved', asyncTest(async () => { + const n1 = await Note.save({ title: 'hello' }); + const noteId = n1.id; + + await sleep(0.1); + + // Simulate the revision service being installed now. There N1 is like an old + // note that had been created before the service existed. + Setting.setValue('revisionService.installedTime', Date.now()); + + // A revision is created the first time a note is overwritten with new content, and + // if this note doesn't already have an existing revision. + // This is mostly to handle old notes that existed before the revision service. If these + // old notes are changed, there's a chance it's accidental or due to some bug, so we + // want to preserve a revision just in case. + + { + await Note.save({ id: noteId, title: 'hello 2' }); + await revisionService().collectRevisions(); // Rev for old note created + Rev for new note + const all = await Revision.allByType(BaseModel.TYPE_NOTE, noteId); + expect(all.length).toBe(2); + const revNote1 = await revisionService().revisionNote(all, 0); + const revNote2 = await revisionService().revisionNote(all, 1); + expect(revNote1.title).toBe('hello'); + expect(revNote2.title).toBe('hello 2'); + } + + // If the note is saved a third time, we don't automatically create a revision. One + // will be created x minutes later when the service collects revisions. + + { + await Note.save({ id: noteId, title: 'hello 3' }); + const all = await Revision.allByType(BaseModel.TYPE_NOTE, noteId); + expect(all.length).toBe(2); + } + })); + + it('should create a revision for notes that get deleted (recyle bin)', asyncTest(async () => { + const n1 = await Note.save({ title: 'hello' }); + const noteId = n1.id; + + await Note.delete(noteId); + + await revisionService().collectRevisions(); + + const all = await Revision.allByType(BaseModel.TYPE_NOTE, noteId); + expect(all.length).toBe(1); + const rev1 = await revisionService().revisionNote(all, 0); + expect(rev1.title).toBe('hello'); + })); + + it('should not create a revision for notes that get deleted if there is already a revision', asyncTest(async () => { + const n1 = await Note.save({ title: 'hello' }); + await revisionService().collectRevisions(); + const noteId = n1.id; + await Note.save({ id: noteId, title: 'hello Paul' }); + await revisionService().collectRevisions(); // REV 1 + + expect((await Revision.allByType(BaseModel.TYPE_NOTE, n1.id)).length).toBe(1); + + await Note.delete(noteId); + + // At this point there is no need to create a new revision for the deleted note + // because we already have the latest version as REV 1 + await revisionService().collectRevisions(); + + expect((await Revision.allByType(BaseModel.TYPE_NOTE, n1.id)).length).toBe(1); + })); + + it('should not create a revision for new note the first time they are saved', asyncTest(async () => { + const n1 = await Note.save({ title: 'hello' }); + + { + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(revisions.length).toBe(0); + } + + await revisionService().collectRevisions(); + + { + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(revisions.length).toBe(0); + } + })); + + it('should abort collecting revisions when one of them is encrypted', asyncTest(async () => { + const n1 = await Note.save({ title: 'hello' }); // CHANGE 1 + await revisionService().collectRevisions(); + await Note.save({ id: n1.id, title: 'hello Ringo' }); // CHANGE 2 + await revisionService().collectRevisions(); + await Note.save({ id: n1.id, title: 'hello George' }); // CHANGE 3 + await revisionService().collectRevisions(); + + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(revisions.length).toBe(2); + + const encryptedRevId = revisions[0].id; + + // Simulate receiving an encrypted revision + await Revision.save({ id: encryptedRevId, encryption_applied: 1 }); + await Note.save({ id: n1.id, title: 'hello Paul' }); // CHANGE 4 + + await revisionService().collectRevisions(); + + // Although change 4 is a note update, check that it has not been processed + // by the collector, due to one of the revisions being encrypted. + expect(await ItemChange.lastChangeId()).toBe(4); + expect(Setting.value('revisionService.lastProcessedChangeId')).toBe(3); + + // Simulate the revision being decrypted by DecryptionService + await Revision.save({ id: encryptedRevId, encryption_applied: 0 }); + + await revisionService().collectRevisions(); + + // Now that the revision has been decrypted, all the changes can be processed + expect(await ItemChange.lastChangeId()).toBe(4); + expect(Setting.value('revisionService.lastProcessedChangeId')).toBe(4); + })); + + it('should not delete old revisions if one of them is still encrypted (1)', asyncTest(async () => { + // Test case 1: Two revisions and the first one is encrypted. + // Calling deleteOldRevisions() with low TTL, which means all revisions + // should be deleted, but they won't be due to the encrypted one. + + const n1_v0 = await Note.save({ title: '' }); + const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' }); + await revisionService().collectRevisions(); // REV 1 + await time.sleep(0.1); + const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' }); + await revisionService().collectRevisions(); // REV 2 + await time.sleep(0.1); + + expect((await Revision.all()).length).toBe(2); + + const revisions = await Revision.all(); + await Revision.save({ id: revisions[0].id, encryption_applied: 1 }); + + await revisionService().deleteOldRevisions(0); + expect((await Revision.all()).length).toBe(2); + + await Revision.save({ id: revisions[0].id, encryption_applied: 0 }); + + await revisionService().deleteOldRevisions(0); + expect((await Revision.all()).length).toBe(0); + })); + + it('should not delete old revisions if one of them is still encrypted (2)', asyncTest(async () => { + // Test case 2: Two revisions and the first one is encrypted. + // Calling deleteOldRevisions() with higher TTL, which means the oldest + // revision should be deleted, but it won't be due to the encrypted one. + + const n1_v0 = await Note.save({ title: '' }); + const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' }); + await revisionService().collectRevisions(); // REV 1 + await time.sleep(0.5); + const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' }); + await revisionService().collectRevisions(); // REV 2 + + expect((await Revision.all()).length).toBe(2); + + const revisions = await Revision.all(); + await Revision.save({ id: revisions[0].id, encryption_applied: 1 }); + + await revisionService().deleteOldRevisions(500); + expect((await Revision.all()).length).toBe(2); + })); + + it('should not delete old revisions if one of them is still encrypted (3)', asyncTest(async () => { + // Test case 2: Two revisions and the second one is encrypted. + // Calling deleteOldRevisions() with higher TTL, which means the oldest + // revision should be deleted, but it won't be due to the encrypted one. + + const n1_v0 = await Note.save({ title: '' }); + const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' }); + await revisionService().collectRevisions(); // REV 1 + await time.sleep(0.5); + const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' }); + await revisionService().collectRevisions(); // REV 2 + + expect((await Revision.all()).length).toBe(2); + + const revisions = await Revision.all(); + await Revision.save({ id: revisions[1].id, encryption_applied: 1 }); + + await revisionService().deleteOldRevisions(500); + expect((await Revision.all()).length).toBe(2); + + await Revision.save({ id: revisions[1].id, encryption_applied: 0 }); + + await revisionService().deleteOldRevisions(500); + expect((await Revision.all()).length).toBe(1); + })); + +}); \ No newline at end of file diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index db7141d8c..9f8220885 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -1,7 +1,7 @@ require('app-module-path').addPath(__dirname); const { time } = require('lib/time-utils.js'); -const { setupDatabase, allSyncTargetItemsEncrypted, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, checkThrowAsync, asyncTest } = require('test-utils.js'); +const { setupDatabase, allSyncTargetItemsEncrypted, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, checkThrowAsync, asyncTest } = require('test-utils.js'); const { shim } = require('lib/shim.js'); const fs = require('fs-extra'); const Folder = require('lib/models/Folder.js'); @@ -13,6 +13,7 @@ const { Database } = require('lib/database.js'); const Setting = require('lib/models/Setting.js'); const MasterKey = require('lib/models/MasterKey'); const BaseItem = require('lib/models/BaseItem.js'); +const Revision = require('lib/models/Revision.js'); const BaseModel = require('lib/BaseModel.js'); const SyncTargetRegistry = require('lib/SyncTargetRegistry.js'); const WelcomeUtils = require('lib/WelcomeUtils'); @@ -23,19 +24,40 @@ process.on('unhandledRejection', (reason, p) => { jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000 + 30000; // The first test is slow because the database needs to be built -async function allItems() { +async function allNotesFolders() { let folders = await Folder.all(); let notes = await Note.all(); return folders.concat(notes); } -async function localItemsSameAsRemote(locals, expect) { +async function remoteItemsByTypes(types) { + const list = await fileApi().list(); + if (list.has_more) throw new Error('Not implemented!!!'); + const files = list.items; + + const output = []; + for (const file of files) { + const remoteContent = await fileApi().get(file.path); + const content = await BaseItem.unserialize(remoteContent); + if (types.indexOf(content.type_) < 0) continue; + output.push(content); + } + return output; +} + +async function remoteNotesAndFolders() { + return remoteItemsByTypes([BaseModel.TYPE_NOTE, BaseModel.TYPE_FOLDER]); +} + +async function remoteNotesFoldersResources() { + return remoteItemsByTypes([BaseModel.TYPE_NOTE, BaseModel.TYPE_FOLDER, BaseModel.TYPE_RESOURCE]); +} + +async function localNotesFoldersSameAsRemote(locals, expect) { let error = null; try { - let files = await fileApi().list(); - files = files.items; - - expect(locals.length).toBe(files.length); + const nf = await remoteNotesAndFolders(); + expect(locals.length).toBe(nf.length); for (let i = 0; i < locals.length; i++) { let dbItem = locals[i]; @@ -45,12 +67,6 @@ async function localItemsSameAsRemote(locals, expect) { expect(!!remote).toBe(true); if (!remote) continue; - // if (syncTargetId() == SyncTargetRegistry.nameToId('filesystem')) { - // expect(remote.updated_time).toBe(Math.floor(dbItem.updated_time / 1000) * 1000); - // } else { - // expect(remote.updated_time).toBe(dbItem.updated_time); - // } - let remoteContent = await fileApi().get(path); remoteContent = dbItem.type_ == BaseModel.TYPE_NOTE ? await Note.unserialize(remoteContent) : await Folder.unserialize(remoteContent); @@ -82,11 +98,11 @@ describe('Synchronizer', function() { let folder = await Folder.save({ title: "folder1" }); await Note.save({ title: "un", parent_id: folder.id }); - let all = await allItems(); + let all = await allNotesFolders(); await synchronizer().start(); - await localItemsSameAsRemote(all, expect); + await localNotesFoldersSameAsRemote(all, expect); })); it('should update remote items', asyncTest(async () => { @@ -96,10 +112,10 @@ describe('Synchronizer', function() { await Note.save({ title: "un UPDATE", id: note.id }); - let all = await allItems(); + let all = await allNotesFolders(); await synchronizer().start(); - await localItemsSameAsRemote(all, expect); + await localNotesFoldersSameAsRemote(all, expect); })); it('should create local items', asyncTest(async () => { @@ -111,9 +127,9 @@ describe('Synchronizer', function() { await synchronizer().start(); - let all = await allItems(); + let all = await allNotesFolders(); - await localItemsSameAsRemote(all, expect); + await localNotesFoldersSameAsRemote(all, expect); })); it('should update local items', asyncTest(async () => { @@ -138,9 +154,9 @@ describe('Synchronizer', function() { await synchronizer().start(); - let all = await allItems(); + let all = await allNotesFolders(); - await localItemsSameAsRemote(all, expect); + await localNotesFoldersSameAsRemote(all, expect); })); it('should resolve note conflicts', asyncTest(async () => { @@ -232,11 +248,9 @@ describe('Synchronizer', function() { await synchronizer().start(); - let files = await fileApi().list(); - files = files.items; - - expect(files.length).toBe(1); - expect(files[0].path).toBe(Folder.systemPath(folder1)); + const remotes = await remoteNotesAndFolders(); + expect(remotes.length).toBe(1); + expect(remotes[0].id).toBe(folder1.id); let deletedItems = await BaseItem.deletedItems(syncTargetId()); expect(deletedItems.length).toBe(0); @@ -279,7 +293,7 @@ describe('Synchronizer', function() { await switchClient(1); context1 = await synchronizer().start({ context: context1 }); - let items = await allItems(); + let items = await allNotesFolders(); expect(items.length).toBe(2); let deletedItems = await BaseItem.deletedItems(syncTargetId()); expect(deletedItems.length).toBe(0); @@ -302,8 +316,8 @@ describe('Synchronizer', function() { await synchronizer().start(); - let all = await allItems(); - await localItemsSameAsRemote(all, expect); + let all = await allNotesFolders(); + await localNotesFoldersSameAsRemote(all, expect); })); it('should delete local folder', asyncTest(async () => { @@ -320,8 +334,8 @@ describe('Synchronizer', function() { await switchClient(1); await synchronizer().start({ context: context1 }); - let items = await allItems(); - await localItemsSameAsRemote(items, expect); + let items = await allNotesFolders(); + await localNotesFoldersSameAsRemote(items, expect); })); it('should resolve conflict if remote folder has been deleted, but note has been added to folder locally', asyncTest(async () => { @@ -338,7 +352,7 @@ describe('Synchronizer', function() { let note = await Note.save({ title: "note1", parent_id: folder1.id }); await synchronizer().start(); - let items = await allItems(); + let items = await allNotesFolders(); expect(items.length).toBe(1); expect(items[0].title).toBe('note1'); expect(items[0].is_conflict).toBe(1); @@ -360,11 +374,11 @@ describe('Synchronizer', function() { await Note.delete(note.id); await synchronizer().start(); - let items = await allItems(); + let items = await allNotesFolders(); expect(items.length).toBe(1); expect(items[0].title).toBe('folder'); - await localItemsSameAsRemote(items, expect); + await localNotesFoldersSameAsRemote(items, expect); })); it('should cross delete all folders', asyncTest(async () => { @@ -393,13 +407,13 @@ describe('Synchronizer', function() { await synchronizer().start(); - let items2 = await allItems(); + let items2 = await allNotesFolders(); await switchClient(1); await synchronizer().start(); - let items1 = await allItems(); + let items1 = await allNotesFolders(); expect(items1.length).toBe(0); expect(items1.length).toBe(items2.length); @@ -462,7 +476,7 @@ describe('Synchronizer', function() { await synchronizer().start(); - let items = await allItems(); + let items = await allNotesFolders(); expect(items.length).toBe(1); })); @@ -680,7 +694,7 @@ describe('Synchronizer', function() { let disabledItems = await BaseItem.syncDisabledItems(syncTargetId()); expect(disabledItems.length).toBe(0); await Note.save({ id: noteId, title: "un mod", }); - synchronizer().testingHooks_ = ['rejectedByTarget']; + synchronizer().testingHooks_ = ['notesRejectedByTarget']; await synchronizer().start(); synchronizer().testingHooks_ = []; await synchronizer().start(); // Another sync to check that this item is now excluded from sync @@ -833,8 +847,8 @@ describe('Synchronizer', function() { await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg'); let resource1 = (await Resource.all())[0]; let resourcePath1 = Resource.fullPath(resource1); - await synchronizer().start(); - expect((await fileApi().list()).items.length).toBe(3); + await synchronizer().start(); + expect((await remoteNotesFoldersResources()).length).toBe(3); await switchClient(2); @@ -901,11 +915,10 @@ describe('Synchronizer', function() { let allResources = await Resource.all(); expect(allResources.length).toBe(1); let all = await fileApi().list(); - expect(all.items.length).toBe(3); + expect((await remoteNotesFoldersResources()).length).toBe(3); await Resource.delete(resource1.id); await synchronizer().start(); - all = await fileApi().list(); - expect(all.items.length).toBe(2); + expect((await remoteNotesFoldersResources()).length).toBe(2); await switchClient(1); @@ -1036,11 +1049,11 @@ describe('Synchronizer', function() { it('should create remote items with UTF-8 content', asyncTest(async () => { let folder = await Folder.save({ title: "Fahrräder" }); await Note.save({ title: "Fahrräder", body: "Fahrräder", parent_id: folder.id }); - let all = await allItems(); + let all = await allNotesFolders(); await synchronizer().start(); - await localItemsSameAsRemote(all, expect); + await localNotesFoldersSameAsRemote(all, expect); })); it("should update remote items but not pull remote changes", asyncTest(async () => { @@ -1058,7 +1071,7 @@ describe('Synchronizer', function() { await Note.save({ title: "un UPDATE", id: note.id }); await synchronizer().start({ syncSteps: ["update_remote"] }); - let all = await allItems(); + let all = await allNotesFolders(); expect(all.length).toBe(2); await switchClient(2); @@ -1110,4 +1123,133 @@ describe('Synchronizer', function() { expect(tags.length).toBe(2); })); + it("should not save revisions when updating a note via sync", asyncTest(async () => { + // When a note is updated, a revision of the original is created. + // Here, on client 1, the note is updated for the first time, however since it is + // via sync, we don't create a revision - that revision has already been created on client + // 2 and is going to be synced. + + const n1 = await Note.save({ title: 'testing' }); + await synchronizer().start(); + + await switchClient(2); + + await synchronizer().start(); + await Note.save({ id: n1.id, title: 'mod from client 2' }); + await revisionService().collectRevisions(); + const allRevs1 = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(allRevs1.length).toBe(1); + await synchronizer().start(); + + await switchClient(1); + + await synchronizer().start(); + const allRevs2 = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(allRevs2.length).toBe(1); + expect(allRevs2[0].id).toBe(allRevs1[0].id); + })); + + it("should not save revisions when deleting a note via sync", asyncTest(async () => { + const n1 = await Note.save({ title: 'testing' }); + await synchronizer().start(); + + await switchClient(2); + + await synchronizer().start(); + await Note.delete(n1.id); + await revisionService().collectRevisions(); // REV 1 + { + const allRevs = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(allRevs.length).toBe(1); + } + await synchronizer().start(); + + await switchClient(1); + + await synchronizer().start(); // The local note gets deleted here, however a new rev is *not* created + { + const allRevs = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(allRevs.length).toBe(1); + } + + const notes = await Note.all(); + expect(notes.length).toBe(0); + })); + + it("should not save revisions when an item_change has been generated as a result of a sync", asyncTest(async () => { + // When a note is modified an item_change object is going to be created. This + // is used for example to tell the search engine, when note should be indexed. It is + // also used by the revision service to tell what note should get a new revision. + // When a note is modified via sync, this item_change object is also created. The issue + // is that we don't want to create revisions for these particular item_changes, because + // such revision has already been created on another client (whatever client initially + // modified the note), and that rev is going to be synced. + // + // So in the end we need to make sure that we don't create these unecessary additional revisions. + + const n1 = await Note.save({ title: 'testing' }); + await synchronizer().start(); + + await switchClient(2); + + await synchronizer().start(); + await Note.save({ id: n1.id, title: 'mod from client 2' }); + await revisionService().collectRevisions(); + await synchronizer().start(); + + await switchClient(1); + + await synchronizer().start(); + + { + const allRevs = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(allRevs.length).toBe(1); + } + + await revisionService().collectRevisions(); + + { + const allRevs = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(allRevs.length).toBe(1); + } + })); + + it("should handle case when new rev is created on client, then older rev arrives later via sync", asyncTest(async () => { + // - C1 creates note 1 + // - C1 modifies note 1 - REV1 created + // - C1 sync + // - C2 sync + // - C2 receives note 1 + // - C2 modifies note 1 - REV2 created (but not based on REV1) + // - C2 receives REV1 + // + // In that case, we need to make sure that REV1 and REV2 are both valid and can be retrieved. + // Even though REV1 was created before REV2, REV2 is *not* based on REV1. This is not ideal + // due to unecessary data being saved, but a possible edge case and we simply need to check + // all the data is valid. + + const n1 = await Note.save({ title: 'note' }); + await Note.save({ id: n1.id, title: 'note REV1' }); + await revisionService().collectRevisions(); // REV1 + expect((await Revision.allByType(BaseModel.TYPE_NOTE, n1.id)).length).toBe(1); + await synchronizer().start(); + + await switchClient(2); + + synchronizer().testingHooks_ = ['skipRevisions']; + await synchronizer().start(); + synchronizer().testingHooks_ = []; + + await Note.save({ id: n1.id, title: 'note REV2' }); + await revisionService().collectRevisions(); // REV2 + expect((await Revision.allByType(BaseModel.TYPE_NOTE, n1.id)).length).toBe(1); + await synchronizer().start(); // Sync the rev that had been skipped above with skipRevisions + + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1.id); + expect(revisions.length).toBe(2); + + expect((await revisionService().revisionNote(revisions, 0)).title).toBe('note REV1'); + expect((await revisionService().revisionNote(revisions, 1)).title).toBe('note REV2'); + })); + }); diff --git a/CliClient/tests/test-utils.js b/CliClient/tests/test-utils.js index 86a81ba11..74c882b04 100644 --- a/CliClient/tests/test-utils.js +++ b/CliClient/tests/test-utils.js @@ -8,6 +8,7 @@ const ItemChange = require('lib/models/ItemChange.js'); const Resource = require('lib/models/Resource.js'); const Tag = require('lib/models/Tag.js'); const NoteTag = require('lib/models/NoteTag.js'); +const Revision = require('lib/models/Revision.js'); const { Logger } = require('lib/logger.js'); const Setting = require('lib/models/Setting.js'); const MasterKey = require('lib/models/MasterKey'); @@ -31,12 +32,14 @@ const SyncTargetDropbox = require('lib/SyncTargetDropbox.js'); const EncryptionService = require('lib/services/EncryptionService.js'); const DecryptionWorker = require('lib/services/DecryptionWorker.js'); const ResourceService = require('lib/services/ResourceService.js'); +const RevisionService = require('lib/services/RevisionService.js'); const WebDavApi = require('lib/WebDavApi'); const DropboxApi = require('lib/DropboxApi'); let databases_ = []; let synchronizers_ = []; let encryptionServices_ = []; +let revisionServices_ = []; let decryptionWorkers_ = []; let resourceServices_ = []; let fileApi_ = null; @@ -82,6 +85,7 @@ BaseItem.loadClass('Resource', Resource); BaseItem.loadClass('Tag', Tag); BaseItem.loadClass('NoteTag', NoteTag); BaseItem.loadClass('MasterKey', MasterKey); +BaseItem.loadClass('Revision', Revision); Setting.setConstant('appId', 'net.cozic.joplin-cli'); Setting.setConstant('appType', 'cli'); @@ -118,6 +122,7 @@ async function switchClient(id) { BaseItem.encryptionService_ = encryptionServices_[id]; Resource.encryptionService_ = encryptionServices_[id]; + BaseItem.revisionService_ = revisionServices_[id]; Setting.setConstant('resourceDir', resourceDir(id)); @@ -129,21 +134,28 @@ async function clearDatabase(id = null) { await ItemChange.waitForAllSaved(); - let queries = [ - 'DELETE FROM notes', - 'DELETE FROM folders', - 'DELETE FROM resources', - 'DELETE FROM tags', - 'DELETE FROM note_tags', - 'DELETE FROM master_keys', - 'DELETE FROM item_changes', - 'DELETE FROM note_resources', - 'DELETE FROM settings', - 'DELETE FROM deleted_items', - 'DELETE FROM sync_items', - 'DELETE FROM notes_normalized', + const tableNames = [ + 'notes', + 'folders', + 'resources', + 'tags', + 'note_tags', + 'master_keys', + 'item_changes', + 'note_resources', + 'settings', + 'deleted_items', + 'sync_items', + 'notes_normalized', + 'revisions', ]; + const queries = []; + for (const n of tableNames) { + queries.push('DELETE FROM ' + n); + queries.push('DELETE FROM sqlite_sequence WHERE name="' + n + '"'); // Reset autoincremented IDs + } + await databases_[id].transactionExecBatch(queries); } @@ -168,6 +180,7 @@ async function setupDatabase(id = null) { }; databases_[id] = new JoplinDatabase(new DatabaseDriverNode()); + databases_[id].setLogger(logger); await databases_[id].open({ name: filePath }); BaseModel.db_ = databases_[id]; @@ -200,6 +213,7 @@ async function setupDatabaseAndSynchronizer(id = null) { } encryptionServices_[id] = new EncryptionService(); + revisionServices_[id] = new RevisionService(); decryptionWorkers_[id] = new DecryptionWorker(); decryptionWorkers_[id].setEncryptionService(encryptionServices_[id]); resourceServices_[id] = new ResourceService(); @@ -222,6 +236,11 @@ function encryptionService(id = null) { return encryptionServices_[id]; } +function revisionService(id = null) { + if (id === null) id = currentClient_; + return revisionServices_[id]; +} + function decryptionWorker(id = null) { if (id === null) id = currentClient_; return decryptionWorkers_[id]; @@ -354,4 +373,4 @@ async function allSyncTargetItemsEncrypted() { return totalCount === encryptedCount; } -module.exports = { resourceService, allSyncTargetItemsEncrypted, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, asyncTest }; \ No newline at end of file +module.exports = { resourceService, allSyncTargetItemsEncrypted, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, asyncTest }; diff --git a/ElectronClient/app/ElectronAppWrapper.js b/ElectronClient/app/ElectronAppWrapper.js index 42c728c84..eb2094f7f 100644 --- a/ElectronClient/app/ElectronAppWrapper.js +++ b/ElectronClient/app/ElectronAppWrapper.js @@ -81,7 +81,7 @@ class ElectronAppWrapper { })) // Uncomment this to view errors if the application does not start - // if (this.env_ === 'dev') this.win_.webContents.openDevTools(); + if (this.env_ === 'dev') this.win_.webContents.openDevTools(); this.win_.on('close', (event) => { // If it's on macOS, the app is completely closed only if the user chooses to close the app (willQuitApp_ will be true) diff --git a/ElectronClient/app/app.js b/ElectronClient/app/app.js index fe65c652f..301d83d03 100644 --- a/ElectronClient/app/app.js +++ b/ElectronClient/app/app.js @@ -29,6 +29,7 @@ const { bridge } = require('electron').remote.require('./bridge'); const Menu = bridge().Menu; const MenuItem = bridge().MenuItem; const PluginManager = require('lib/services/PluginManager'); +const RevisionService = require('lib/services/RevisionService'); const pluginClasses = [ require('./plugins/GotoAnything.min'), @@ -1031,6 +1032,11 @@ class Application extends BaseApplication { ExternalEditWatcher.instance().setLogger(reg.logger()); ExternalEditWatcher.instance().dispatch = this.store().dispatch; + + RevisionService.instance().runInBackground(); + + // Make it available to the console window - useful to call revisionService.collectRevisions() + window.revisionService = RevisionService.instance(); } } diff --git a/ElectronClient/app/gui/ConfigScreen.jsx b/ElectronClient/app/gui/ConfigScreen.jsx index f09034da5..61644e79c 100644 --- a/ElectronClient/app/gui/ConfigScreen.jsx +++ b/ElectronClient/app/gui/ConfigScreen.jsx @@ -265,9 +265,12 @@ class ConfigScreenComponent extends React.Component { updateSettingValue(key, event.target.value); }; + const label = [md.label()]; + if (md.unitLabel) label.push('(' + md.unitLabel() + ')'); + return (
-
+
{onNumChange(event)}} min={md.minimum} max={md.maximum} step={md.step}/> { descriptionComp }
diff --git a/ElectronClient/app/gui/HelpButton.jsx b/ElectronClient/app/gui/HelpButton.jsx new file mode 100644 index 000000000..7389c8884 --- /dev/null +++ b/ElectronClient/app/gui/HelpButton.jsx @@ -0,0 +1,39 @@ +const React = require('react'); +const { connect } = require('react-redux'); +const { reg } = require('lib/registry.js'); +const { themeStyle } = require('../theme.js'); +const { _ } = require('lib/locale.js'); +const { bridge } = require('electron').remote.require('./bridge'); + +class HelpButtonComponent extends React.Component { + + constructor() { + super(); + + this.onClick = this.onClick.bind(this); + } + + onClick() { + if (this.props.onClick) this.props.onClick(); + } + + render() { + const theme = themeStyle(this.props.theme); + let style = Object.assign({}, this.props.style, {color: theme.color, textDecoration: 'none'}); + const helpIconStyle = {flex:0, width: 16, height: 16, marginLeft: 10}; + const extraProps = {}; + if (this.props.tip) extraProps['data-tip'] = this.props.tip; + return + } + +} + +const mapStateToProps = (state) => { + return { + theme: state.settings.theme, + }; +}; + +const HelpButton = connect(mapStateToProps)(HelpButtonComponent); + +module.exports = HelpButton; diff --git a/ElectronClient/app/gui/MainScreen.jsx b/ElectronClient/app/gui/MainScreen.jsx index 215721bf3..d9fb93e62 100644 --- a/ElectronClient/app/gui/MainScreen.jsx +++ b/ElectronClient/app/gui/MainScreen.jsx @@ -227,6 +227,7 @@ class MainScreenComponent extends React.Component { notePropertiesDialogOptions: { noteId: command.noteId, visible: true, + onRevisionLinkClick: command.onRevisionLinkClick, }, }); } else if (command.name === 'toggleVisiblePanes') { @@ -474,6 +475,7 @@ class MainScreenComponent extends React.Component { theme={this.props.theme} noteId={notePropertiesDialogOptions.noteId} onClose={this.notePropertiesDialog_close} + onRevisionLinkClick={notePropertiesDialogOptions.onRevisionLinkClick} /> } bridge().openExternal(url)} style={theme.urlStyle}>{displayedValue} + } else if (key === 'revisionsLink') { + controlComp = {_('Previous versions of this note')} } else { controlComp =
{displayedValue}
} - if (key !== 'id') { + if (key !== 'id' && key !== 'revisionsLink') { editCompHandler = () => {this.editPropertyButtonClick(key, value)}; editCompIcon = 'fa-edit'; } diff --git a/ElectronClient/app/gui/NoteRevisionViewer.jsx b/ElectronClient/app/gui/NoteRevisionViewer.jsx new file mode 100644 index 000000000..2e6610acd --- /dev/null +++ b/ElectronClient/app/gui/NoteRevisionViewer.jsx @@ -0,0 +1,175 @@ +const React = require('react'); +const { connect } = require('react-redux'); +const { themeStyle } = require('../theme.js'); +const { _ } = require('lib/locale.js'); +const NoteTextViewer = require('./NoteTextViewer.min'); +const HelpButton = require('./HelpButton.min'); +const BaseModel = require('lib/BaseModel'); +const Revision = require('lib/models/Revision'); +const Setting = require('lib/models/Setting'); +const RevisionService = require('lib/services/RevisionService'); +const shared = require('lib/components/shared/note-screen-shared.js'); +const MdToHtml = require('lib/MdToHtml'); +const { time } = require('lib/time-utils.js'); +const ReactTooltip = require('react-tooltip'); +const { substrWithEllipsis } = require('lib/string-utils'); + +class NoteRevisionViewerComponent extends React.PureComponent { + + constructor() { + super(); + + this.state = { + revisions: [], + currentRevId: '', + note: null, + restoring: false, + }; + + this.viewerRef_ = React.createRef(); + + this.viewer_domReady = this.viewer_domReady.bind(this); + this.revisionList_onChange = this.revisionList_onChange.bind(this); + this.importButton_onClick = this.importButton_onClick.bind(this); + this.backButton_click = this.backButton_click.bind(this); + } + + style() { + const theme = themeStyle(this.props.theme); + + let style = { + root: { + backgroundColor: theme.backgroundColor, + display: 'flex', + flex: 1, + flexDirection: 'column', + }, + titleInput: Object.assign({}, theme.inputStyle, { flex: 1 }), + revisionList: Object.assign({}, theme.dropdownList, { marginLeft: 10, flex: 0.5 }), + }; + + return style; + } + + async viewer_domReady() { + // this.viewerRef_.current.wrappedInstance.openDevTools(); + + const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, this.props.noteId); + + this.setState({ + revisions: revisions, + currentRevId: revisions.length ? revisions[revisions.length - 1].id : '', + }, () => { + this.reloadNote(); + }); + } + + async importButton_onClick() { + if (!this.state.note) return; + this.setState({ restoring: true }); + await RevisionService.instance().importRevisionNote(this.state.note); + this.setState({ restoring: false }); + alert(_('The note "%s" has been successfully restored to the notebook "%s".', substrWithEllipsis(this.state.note.title, 0, 32), RevisionService.instance().restoreFolderTitle())); + } + + backButton_click() { + if (this.props.onBack) this.props.onBack(); + } + + revisionList_onChange(event) { + const value = event.target.value; + + if (!value) { + if (this.props.onBack) this.props.onBack(); + } else { + this.setState({ + currentRevId: value, + }, () => { + this.reloadNote(); + }); + } + } + + async reloadNote() { + let noteBody = ''; + if (!this.state.revisions.length || !this.state.currentRevId) { + noteBody = _('This note has no history'); + this.setState({ note: null }); + } else { + const revIndex = BaseModel.modelIndexById(this.state.revisions, this.state.currentRevId); + const note = await RevisionService.instance().revisionNote(this.state.revisions, revIndex); + if (!note) return; + noteBody = note.body; + this.setState({ note: note }); + } + + const theme = themeStyle(this.props.theme); + + const mdToHtml = new MdToHtml({ + resourceBaseUrl: 'file://' + Setting.value('resourceDir') + '/', + }); + + const result = mdToHtml.render(noteBody, theme, { + codeTheme: theme.codeThemeCss, + userCss: this.props.customCss ? this.props.customCss : '', + resources: await shared.attachedResources(noteBody), + }); + + this.viewerRef_.current.wrappedInstance.send('setHtml', result.html, { cssFiles: result.cssFiles }); + } + + render() { + const theme = themeStyle(this.props.theme); + const style = this.style(); + + const revisionListItems = []; + const revs = this.state.revisions.slice().reverse(); + for (let i = 0; i < revs.length; i++) { + const rev = revs[i]; + revisionListItems.push(); + } + + const restoreButtonTitle = _('Restore'); + const helpMessage = _('Click "%s" to restore the note. It will be copied in the notebook named "%s". The current version of the note will not be replaced or modified.', restoreButtonTitle, RevisionService.instance().restoreFolderTitle()); + + const titleInput = ( +
+ + + + + +
+ ); + + const viewer = + + return ( +
+ {titleInput} + {viewer} + +
+ ); + } + +} + +const mapStateToProps = (state) => { + return { + theme: state.settings.theme, + }; +}; + +const NoteRevisionViewer = connect(mapStateToProps)(NoteRevisionViewerComponent); + +module.exports = NoteRevisionViewer; \ No newline at end of file diff --git a/ElectronClient/app/gui/NoteText.jsx b/ElectronClient/app/gui/NoteText.jsx index 5c07733f5..fa197ff68 100644 --- a/ElectronClient/app/gui/NoteText.jsx +++ b/ElectronClient/app/gui/NoteText.jsx @@ -38,6 +38,7 @@ const { clipboard } = require('electron'); const SearchEngine = require('lib/services/SearchEngine'); const ModelCache = require('lib/services/ModelCache'); const NoteTextViewer = require('./NoteTextViewer.min'); +const NoteRevisionViewer = require('./NoteRevisionViewer.min'); require('brace/mode/markdown'); // https://ace.c9.io/build/kitchen-sink.html @@ -70,6 +71,7 @@ class NoteTextComponent extends React.Component { editorScrollTop: 0, newNote: null, noteTags: [], + showRevisions: false, // If the current note was just created, and the title has never been // changed by the user, this variable contains that note ID. Used @@ -268,6 +270,7 @@ class NoteTextComponent extends React.Component { this.titleField_keyDown = this.titleField_keyDown.bind(this); this.webview_ipcMessage = this.webview_ipcMessage.bind(this); this.webview_domReady = this.webview_domReady.bind(this); + this.noteRevisionViewer_onBack = this.noteRevisionViewer_onBack.bind(this); } // Note: @@ -530,7 +533,8 @@ class NoteTextComponent extends React.Component { webviewReady: webviewReady, folder: parentFolder, lastKeys: [], - noteTags: noteTags + noteTags: noteTags, + showRevisions: false, }; if (!note) { @@ -619,6 +623,13 @@ class NoteTextComponent extends React.Component { return shared.refreshNoteMetadata(this, force); } + async noteRevisionViewer_onBack() { + this.setState({ showRevisions: false }); + + this.lastSetHtml_ = ''; + this.scheduleReloadNote(this.props); + } + title_changeText(event) { shared.noteComponent_change(this, 'title', event.target.value); this.setState({ newAndNoTitleChangeNoteId: null }); @@ -1529,6 +1540,7 @@ class NoteTextComponent extends React.Component { type: 'WINDOW_COMMAND', name: 'commandNoteProperties', noteId: n.id, + onRevisionLinkClick: () => { this.setState({ showRevisions: true}) }, }); }, }); @@ -1610,6 +1622,18 @@ class NoteTextComponent extends React.Component { const innerWidth = rootStyle.width - rootStyle.paddingLeft - rootStyle.paddingRight - borderWidth; + if (this.state.showRevisions && note && note.id) { + rootStyle.paddingRight = rootStyle.paddingLeft; + rootStyle.paddingTop = rootStyle.paddingLeft; + rootStyle.paddingBottom = rootStyle.paddingLeft; + rootStyle.display = 'inline-flex'; + return ( +
+ +
+ ); + } + if (this.props.selectedNoteIds.length > 1) { return this.renderMultiNotes(rootStyle); } else if (!note || !!note.encryption_applied) { //|| (note && !this.props.newNote && this.props.noteId && note.id !== this.props.noteId)) { // note.id !== props.noteId is when the note has not been loaded yet, and the previous one is still in the state @@ -1710,7 +1734,7 @@ class NoteTextComponent extends React.Component { viewerStyle.borderLeft = 'none'; } - if (this.state.webviewReady) { + if (this.state.webviewReady && this.webviewRef_.current) { let html = this.state.bodyHtml; const htmlHasChanged = this.lastSetHtml_ !== html; diff --git a/ElectronClient/app/gui/NoteTextViewer.jsx b/ElectronClient/app/gui/NoteTextViewer.jsx index d691250cf..afefea3c7 100644 --- a/ElectronClient/app/gui/NoteTextViewer.jsx +++ b/ElectronClient/app/gui/NoteTextViewer.jsx @@ -18,11 +18,11 @@ class NoteTextViewerComponent extends React.Component { } webview_domReady(event) { - this.props.onDomReady(event); + if (this.props.onDomReady) this.props.onDomReady(event); } webview_ipcMessage(event) { - this.props.onIpcMessage(event); + if (this.props.onIpcMessage) this.props.onIpcMessage(event); } initWebview() { @@ -67,13 +67,21 @@ class NoteTextViewerComponent extends React.Component { } } - componentDidUpdate() { + tryInit() { if (!this.initialized_ && this.webviewRef_.current) { this.initWebview(); this.initialized_ = true; } } + componentDidMount() { + this.tryInit(); + } + + componentDidUpdate() { + this.tryInit(); + } + componentWillUnmount() { this.destroyWebview(); } diff --git a/ElectronClient/app/gui/SideBar.jsx b/ElectronClient/app/gui/SideBar.jsx index 916a7240c..183bc98e0 100644 --- a/ElectronClient/app/gui/SideBar.jsx +++ b/ElectronClient/app/gui/SideBar.jsx @@ -221,7 +221,7 @@ class SideBarComponent extends React.Component { } } } else if (command.name === 'synchronize') { - this.sync_click(); + if (!this.props.syncStarted) this.sync_click(); } else { commandProcessed = false; } diff --git a/ElectronClient/app/main-html.js b/ElectronClient/app/main-html.js index 7a71c3613..e83dc15cb 100644 --- a/ElectronClient/app/main-html.js +++ b/ElectronClient/app/main-html.js @@ -21,6 +21,7 @@ const Tag = require('lib/models/Tag.js'); const NoteTag = require('lib/models/NoteTag.js'); const MasterKey = require('lib/models/MasterKey'); const Setting = require('lib/models/Setting.js'); +const Revision = require('lib/models/Revision.js'); const { Logger } = require('lib/logger.js'); const { FsDriverNode } = require('lib/fs-driver-node.js'); const { shimInit } = require('lib/shim-init-node.js'); @@ -42,6 +43,7 @@ BaseItem.loadClass('Resource', Resource); BaseItem.loadClass('Tag', Tag); BaseItem.loadClass('NoteTag', NoteTag); BaseItem.loadClass('MasterKey', MasterKey); +BaseItem.loadClass('Revision', Revision); Setting.setConstant('appId', 'net.cozic.joplin-desktop'); Setting.setConstant('appType', 'desktop'); diff --git a/ElectronClient/app/package-lock.json b/ElectronClient/app/package-lock.json index 235447c92..0e4b63f0d 100644 --- a/ElectronClient/app/package-lock.json +++ b/ElectronClient/app/package-lock.json @@ -23,9 +23,9 @@ "optional": true }, "7zip-bin-win": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/7zip-bin-win/-/7zip-bin-win-2.1.1.tgz", - "integrity": "sha512-6VGEW7PXGroTsoI2QW3b0ea95HJmbVBHvfANKLLMzSzFA1zKqVX5ybNuhmeGpf6vA0x8FJTt6twpprDANsY5WQ==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/7zip-bin-win/-/7zip-bin-win-2.2.0.tgz", + "integrity": "sha512-uPHXapEmUtlUKTBx4asWMlxtFUWXzEY0KVEgU7QKhgO2LJzzM3kYxM6yOyUZTtYE6mhK4dDn3FDut9SCQWHzgg==", "optional": true }, "@types/node": { @@ -1214,6 +1214,11 @@ "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", "dev": true }, + "classnames": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.2.6.tgz", + "integrity": "sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q==" + }, "cli-boxes": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", @@ -2357,25 +2362,29 @@ "dependencies": { "abbrev": { "version": "1.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true, "optional": true }, "ansi-regex": { "version": "2.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", "dev": true, "optional": true }, "aproba": { "version": "1.2.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true, "optional": true }, "are-we-there-yet": { "version": "1.1.4", - "bundled": true, + "resolved": false, + "integrity": "sha1-u13KOCu5TwXhUZQ3PRb9O6HKEQ0=", "dev": true, "optional": true, "requires": { @@ -2385,13 +2394,15 @@ }, "balanced-match": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true, "optional": true }, "brace-expansion": { "version": "1.1.11", - "bundled": true, + "resolved": false, + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dev": true, "optional": true, "requires": { @@ -2401,37 +2412,43 @@ }, "chownr": { "version": "1.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE=", "dev": true, "optional": true }, "code-point-at": { "version": "1.1.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", "dev": true, "optional": true }, "concat-map": { "version": "0.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true, "optional": true }, "console-control-strings": { "version": "1.1.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", "dev": true, "optional": true }, "core-util-is": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true, "optional": true }, "debug": { "version": "2.6.9", - "bundled": true, + "resolved": false, + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "dev": true, "optional": true, "requires": { @@ -2440,25 +2457,29 @@ }, "deep-extend": { "version": "0.5.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-N8vBdOa+DF7zkRrDCsaOXoCs/E2fJfx9B9MrKnnSiHNh4ws7eSys6YQE4KvT1cecKmOASYQBhbKjeuDD9lT81w==", "dev": true, "optional": true }, "delegates": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", "dev": true, "optional": true }, "detect-libc": { "version": "1.0.3", - "bundled": true, + "resolved": false, + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", "dev": true, "optional": true }, "fs-minipass": { "version": "1.2.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", "dev": true, "optional": true, "requires": { @@ -2467,13 +2488,15 @@ }, "fs.realpath": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true, "optional": true }, "gauge": { "version": "2.7.4", - "bundled": true, + "resolved": false, + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", "dev": true, "optional": true, "requires": { @@ -2489,7 +2512,8 @@ }, "glob": { "version": "7.1.2", - "bundled": true, + "resolved": false, + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", "dev": true, "optional": true, "requires": { @@ -2503,13 +2527,15 @@ }, "has-unicode": { "version": "2.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", "dev": true, "optional": true }, "iconv-lite": { "version": "0.4.21", - "bundled": true, + "resolved": false, + "integrity": "sha512-En5V9za5mBt2oUA03WGD3TwDv0MKAruqsuxstbMUZaj9W9k/m1CV/9py3l0L5kw9Bln8fdHQmzHSYtvpvTLpKw==", "dev": true, "optional": true, "requires": { @@ -2518,7 +2544,8 @@ }, "ignore-walk": { "version": "3.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==", "dev": true, "optional": true, "requires": { @@ -2527,7 +2554,8 @@ }, "inflight": { "version": "1.0.6", - "bundled": true, + "resolved": false, + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "dev": true, "optional": true, "requires": { @@ -2537,19 +2565,22 @@ }, "inherits": { "version": "2.0.3", - "bundled": true, + "resolved": false, + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", "dev": true, "optional": true }, "ini": { "version": "1.3.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "dev": true, "optional": true }, "is-fullwidth-code-point": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "dev": true, "optional": true, "requires": { @@ -2558,13 +2589,15 @@ }, "isarray": { "version": "1.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", "dev": true, "optional": true }, "minimatch": { "version": "3.0.4", - "bundled": true, + "resolved": false, + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "dev": true, "optional": true, "requires": { @@ -2573,13 +2606,15 @@ }, "minimist": { "version": "0.0.8", - "bundled": true, + "resolved": false, + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", "dev": true, "optional": true }, "minipass": { "version": "2.2.4", - "bundled": true, + "resolved": false, + "integrity": "sha512-hzXIWWet/BzWhYs2b+u7dRHlruXhwdgvlTMDKC6Cb1U7ps6Ac6yQlR39xsbjWJE377YTCtKwIXIpJ5oP+j5y8g==", "dev": true, "optional": true, "requires": { @@ -2589,7 +2624,8 @@ }, "minizlib": { "version": "1.1.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-4T6Ur/GctZ27nHfpt9THOdRZNgyJ9FZchYO1ceg5S8Q3DNLCKYy44nCZzgCJgcvx2UM8czmqak5BCxJMrq37lA==", "dev": true, "optional": true, "requires": { @@ -2598,7 +2634,8 @@ }, "mkdirp": { "version": "0.5.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "optional": true, "requires": { @@ -2607,13 +2644,15 @@ }, "ms": { "version": "2.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true, "optional": true }, "needle": { "version": "2.2.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-eFagy6c+TYayorXw/qtAdSvaUpEbBsDwDyxYFgLZ0lTojfH7K+OdBqAF7TAFwDokJaGpubpSGG0wO3iC0XPi8w==", "dev": true, "optional": true, "requires": { @@ -2624,7 +2663,8 @@ }, "node-pre-gyp": { "version": "0.10.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-G7kEonQLRbcA/mOoFoxvlMrw6Q6dPf92+t/l0DFSMuSlDoWaI9JWIyPwK0jyE1bph//CUEL65/Fz1m2vJbmjQQ==", "dev": true, "optional": true, "requires": { @@ -2642,7 +2682,8 @@ }, "nopt": { "version": "4.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", "dev": true, "optional": true, "requires": { @@ -2652,13 +2693,15 @@ }, "npm-bundled": { "version": "1.0.3", - "bundled": true, + "resolved": false, + "integrity": "sha512-ByQ3oJ/5ETLyglU2+8dBObvhfWXX8dtPZDMePCahptliFX2iIuhyEszyFk401PZUNQH20vvdW5MLjJxkwU80Ow==", "dev": true, "optional": true }, "npm-packlist": { "version": "1.1.10", - "bundled": true, + "resolved": false, + "integrity": "sha512-AQC0Dyhzn4EiYEfIUjCdMl0JJ61I2ER9ukf/sLxJUcZHfo+VyEfz2rMJgLZSS1v30OxPQe1cN0LZA1xbcaVfWA==", "dev": true, "optional": true, "requires": { @@ -2668,7 +2711,8 @@ }, "npmlog": { "version": "4.1.2", - "bundled": true, + "resolved": false, + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "dev": true, "optional": true, "requires": { @@ -2680,19 +2724,22 @@ }, "number-is-nan": { "version": "1.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", "dev": true, "optional": true }, "object-assign": { "version": "4.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "dev": true, "optional": true }, "once": { "version": "1.4.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, "optional": true, "requires": { @@ -2701,19 +2748,22 @@ }, "os-homedir": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true, "optional": true }, "os-tmpdir": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true, "optional": true }, "osenv": { "version": "0.1.5", - "bundled": true, + "resolved": false, + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", "dev": true, "optional": true, "requires": { @@ -2723,19 +2773,22 @@ }, "path-is-absolute": { "version": "1.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true, "optional": true }, "process-nextick-args": { "version": "2.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", "dev": true, "optional": true }, "rc": { "version": "1.2.7", - "bundled": true, + "resolved": false, + "integrity": "sha512-LdLD8xD4zzLsAT5xyushXDNscEjB7+2ulnl8+r1pnESlYtlJtVSoCMBGr30eDRJ3+2Gq89jK9P9e4tCEH1+ywA==", "dev": true, "optional": true, "requires": { @@ -2747,7 +2800,8 @@ "dependencies": { "minimist": { "version": "1.2.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true, "optional": true } @@ -2755,7 +2809,8 @@ }, "readable-stream": { "version": "2.3.6", - "bundled": true, + "resolved": false, + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "optional": true, "requires": { @@ -2770,7 +2825,8 @@ }, "rimraf": { "version": "2.6.2", - "bundled": true, + "resolved": false, + "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", "dev": true, "optional": true, "requires": { @@ -2779,43 +2835,50 @@ }, "safe-buffer": { "version": "5.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==", "dev": true, "optional": true }, "safer-buffer": { "version": "2.1.2", - "bundled": true, + "resolved": false, + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true, "optional": true }, "sax": { "version": "1.2.4", - "bundled": true, + "resolved": false, + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", "dev": true, "optional": true }, "semver": { "version": "5.5.0", - "bundled": true, + "resolved": false, + "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", "dev": true, "optional": true }, "set-blocking": { "version": "2.0.0", - "bundled": true, + "resolved": false, + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true, "optional": true }, "signal-exit": { "version": "3.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true, "optional": true }, "string-width": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "optional": true, "requires": { @@ -2826,7 +2889,8 @@ }, "string_decoder": { "version": "1.1.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "optional": true, "requires": { @@ -2835,7 +2899,8 @@ }, "strip-ansi": { "version": "3.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "optional": true, "requires": { @@ -2844,13 +2909,15 @@ }, "strip-json-comments": { "version": "2.0.1", - "bundled": true, + "resolved": false, + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "dev": true, "optional": true }, "tar": { "version": "4.4.1", - "bundled": true, + "resolved": false, + "integrity": "sha512-O+v1r9yN4tOsvl90p5HAP4AEqbYhx4036AGMm075fH9F8Qwi3oJ+v4u50FkT/KkvywNGtwkk0zRI+8eYm1X/xg==", "dev": true, "optional": true, "requires": { @@ -2865,13 +2932,15 @@ }, "util-deprecate": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "dev": true, "optional": true }, "wide-align": { "version": "1.1.2", - "bundled": true, + "resolved": false, + "integrity": "sha512-ijDLlyQ7s6x1JgCLur53osjm/UXUYD9+0PbYKrBsYisYXzCxN+HC3mYDNy/dWdmf3AwqwU3CXwDCvsNgGK1S0w==", "dev": true, "optional": true, "requires": { @@ -2880,13 +2949,15 @@ }, "wrappy": { "version": "1.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true, "optional": true }, "yallist": { "version": "3.0.2", - "bundled": true, + "resolved": false, + "integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k=", "dev": true, "optional": true } @@ -4899,14 +4970,36 @@ } }, "react": { - "version": "16.4.0", - "resolved": "https://registry.npmjs.org/react/-/react-16.4.0.tgz", - "integrity": "sha512-K0UrkLXSAekf5nJu89obKUM7o2vc6MMN9LYoKnCa+c+8MJRAT120xzPLENcWSRc7GYKIg0LlgJRDorrufdglQQ==", + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/react/-/react-16.8.1.tgz", + "integrity": "sha512-wLw5CFGPdo7p/AgteFz7GblI2JPOos0+biSoxf1FPsGxWQZdN/pj6oToJs1crn61DL3Ln7mN86uZ4j74p31ELQ==", "requires": { - "fbjs": "^0.8.16", "loose-envify": "^1.1.0", "object-assign": "^4.1.1", - "prop-types": "^15.6.0" + "prop-types": "^15.6.2", + "scheduler": "^0.13.1" + }, + "dependencies": { + "prop-types": { + "version": "15.7.2", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", + "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.8.1" + }, + "dependencies": { + "loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + } + } + } } }, "react-ace": { @@ -4950,6 +5043,11 @@ "prop-types": "^15.6.0" } }, + "react-is": { + "version": "16.8.6", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.8.6.tgz", + "integrity": "sha512-aUk3bHfZ2bRSVFFbbeVS4i+lNPZr3/WM5jT2J5omUVV1zzcs1nAaf3l51ctA5FFvCRbhrH0bdAsRRQddFJZPtA==" + }, "react-onclickoutside": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/react-onclickoutside/-/react-onclickoutside-6.7.1.tgz", @@ -4975,6 +5073,15 @@ } } }, + "react-tooltip": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/react-tooltip/-/react-tooltip-3.10.0.tgz", + "integrity": "sha512-GGdxJvM1zSFztkTP7gCQbLTstWr1OOoMpJ5WZUGhimj0nhRY+MPz+92MpEnKmj0cftJ9Pd/M6FfSl0sfzmZWkg==", + "requires": { + "classnames": "^2.2.5", + "prop-types": "^15.6.0" + } + }, "read-chunk": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-2.1.0.tgz", @@ -5303,6 +5410,15 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "scheduler": { + "version": "0.13.6", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.13.6.tgz", + "integrity": "sha512-IWnObHt413ucAYKsD9J1QShUKkbKLQQHdxRyw73sw4FN26iWr3DY/H34xGPe4nmL1DwXyWmSWmMrA9TfQbE/XQ==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1" + } + }, "semver": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", diff --git a/ElectronClient/app/package.json b/ElectronClient/app/package.json index 4ead0d9ef..2567299d3 100644 --- a/ElectronClient/app/package.json +++ b/ElectronClient/app/package.json @@ -88,6 +88,7 @@ "chokidar": "^3.0.0", "compare-versions": "^3.2.1", "diacritics": "^1.3.0", + "diff-match-patch": "^1.0.4", "electron-context-menu": "^0.9.1", "electron-is-dev": "^0.3.0", "electron-window-state": "^4.1.1", @@ -132,6 +133,7 @@ "react-datetime": "^2.14.0", "react-dom": "^16.4.0", "react-redux": "^5.0.7", + "react-tooltip": "^3.10.0", "read-chunk": "^2.1.0", "readability-node": "^0.1.0", "redux": "^3.7.2", diff --git a/ElectronClient/app/plugins/GotoAnything.jsx b/ElectronClient/app/plugins/GotoAnything.jsx index bc427d3d3..38d04d08a 100644 --- a/ElectronClient/app/plugins/GotoAnything.jsx +++ b/ElectronClient/app/plugins/GotoAnything.jsx @@ -6,6 +6,7 @@ const SearchEngine = require('lib/services/SearchEngine'); const BaseModel = require('lib/BaseModel'); const Tag = require('lib/models/Tag'); const { ItemList } = require('../gui/ItemList.min'); +const HelpButton = require('../gui/HelpButton.min'); const { substrWithEllipsis, surroundKeywords } = require('lib/string-utils.js'); const PLUGIN_NAME = 'gotoAnything'; @@ -61,8 +62,6 @@ class Dialog extends React.PureComponent { row: {overflow: 'hidden', height:itemHeight, display: 'flex', justifyContent: 'center', flexDirection: 'column', paddingLeft: 10, paddingRight: 10}, help: Object.assign({}, theme.textStyle, { marginBottom: 10 }), inputHelpWrapper: {display: 'flex', flexDirection: 'row', alignItems: 'center'}, - helpIcon: {flex:0, width: 16, height: 16, marginLeft: 10}, - helpButton: {color: theme.color, textDecoration: 'none'}, }; const rowTextStyle = { @@ -321,7 +320,7 @@ class Dialog extends React.PureComponent { {helpComp}
- +
{this.renderList()} diff --git a/ElectronClient/app/style.css b/ElectronClient/app/style.css index d4c8f55ee..a39a653f9 100644 --- a/ElectronClient/app/style.css +++ b/ElectronClient/app/style.css @@ -113,3 +113,12 @@ table td, table th { .note-property-box .rdt { display: inline-block; } + +.help-tooltip { + font-family: sans-serif; + max-width: 200px; +} + +:disabled { + opacity: 0.6; +} \ No newline at end of file diff --git a/ElectronClient/app/theme.js b/ElectronClient/app/theme.js index ce9d2af85..09b2fb1b9 100644 --- a/ElectronClient/app/theme.js +++ b/ElectronClient/app/theme.js @@ -34,6 +34,9 @@ globalStyle.icon = { globalStyle.lineInput = { fontFamily: globalStyle.fontFamily, + maxHeight: 22, + height: 22, + paddingLeft: 5, }; globalStyle.headerStyle = { @@ -43,6 +46,7 @@ globalStyle.headerStyle = { globalStyle.inputStyle = { border: '1px solid', height: 24, + maxHeight: 24, paddingLeft: 5, paddingRight: 5, boxSizing: 'border-box', @@ -54,13 +58,14 @@ globalStyle.containerStyle = { }; globalStyle.buttonStyle = { - marginRight: 10, + // marginRight: 10, border: '1px solid', - minHeight: 30, + minHeight: 26, minWidth: 80, maxWidth: 160, paddingLeft: 12, paddingRight: 12, + boxShadow: '0px 1px 1px rgba(0,0,0,0.3)', }; const lightStyle = { @@ -226,6 +231,8 @@ function addExtraStyles(style) { style.dialogTitle = Object.assign({}, style.h1Style, { marginBottom: '1.2em' }); + style.dropdownList = Object.assign({}, style.inputStyle); + return style; } diff --git a/ReactNativeClient/lib/BaseApplication.js b/ReactNativeClient/lib/BaseApplication.js index fe201ca56..2047aa651 100644 --- a/ReactNativeClient/lib/BaseApplication.js +++ b/ReactNativeClient/lib/BaseApplication.js @@ -35,6 +35,7 @@ const SyncTargetDropbox = require('lib/SyncTargetDropbox.js'); const EncryptionService = require('lib/services/EncryptionService'); const ResourceFetcher = require('lib/services/ResourceFetcher'); const SearchEngineUtils = require('lib/services/SearchEngineUtils'); +const RevisionService = require('lib/services/RevisionService'); const DecryptionWorker = require('lib/services/DecryptionWorker'); const BaseService = require('lib/services/BaseService'); const SearchEngine = require('lib/services/SearchEngine'); @@ -577,6 +578,8 @@ class BaseApplication { time.setDateFormat(Setting.value('dateFormat')); time.setTimeFormat(Setting.value('timeFormat')); + BaseItem.revisionService_ = RevisionService.instance(); + BaseService.logger_ = this.logger_; EncryptionService.instance().setLogger(this.logger_); BaseItem.encryptionService_ = EncryptionService.instance(); diff --git a/ReactNativeClient/lib/BaseModel.js b/ReactNativeClient/lib/BaseModel.js index 9e0d70371..3811a70ae 100644 --- a/ReactNativeClient/lib/BaseModel.js +++ b/ReactNativeClient/lib/BaseModel.js @@ -539,6 +539,7 @@ BaseModel.typeEnum_ = [ ['TYPE_ITEM_CHANGE', 10], ['TYPE_NOTE_RESOURCE', 11], ['TYPE_RESOURCE_LOCAL_STATE', 12], + ['TYPE_REVISION', 13], ]; for (let i = 0; i < BaseModel.typeEnum_.length; i++) { diff --git a/ReactNativeClient/lib/components/screens/config.js b/ReactNativeClient/lib/components/screens/config.js index e02ed1c70..848c07d1d 100644 --- a/ReactNativeClient/lib/components/screens/config.js +++ b/ReactNativeClient/lib/components/screens/config.js @@ -10,7 +10,7 @@ const Setting = require('lib/models/Setting.js'); const shared = require('lib/components/shared/config-shared.js'); const SyncTargetRegistry = require('lib/SyncTargetRegistry'); const { reg } = require('lib/registry.js'); -import VersionInfo from 'react-native-version-info'; +const VersionInfo = require('react-native-version-info').default; class ConfigScreenComponent extends BaseScreenComponent { @@ -163,10 +163,14 @@ class ConfigScreenComponent extends BaseScreenComponent { ); } else if (md.type == Setting.TYPE_INT) { + const unitLabel = md.unitLabel ? md.unitLabel(value) : value; return ( {md.label()} - updateSettingValue(key, value)} /> + + updateSettingValue(key, value)} /> + {unitLabel} + ); } else if (md.type == Setting.TYPE_STRING) { diff --git a/ReactNativeClient/lib/joplin-database.js b/ReactNativeClient/lib/joplin-database.js index 66275dae7..4f8511c6e 100644 --- a/ReactNativeClient/lib/joplin-database.js +++ b/ReactNativeClient/lib/joplin-database.js @@ -263,7 +263,7 @@ class JoplinDatabase extends Database { // must be set in the synchronizer too. // Note: v16 and v17 don't do anything. They were used to debug an issue. - const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]; + const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]; let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion); @@ -515,6 +515,35 @@ class JoplinDatabase extends Database { END;`); } + if (targetVersion == 19) { + const newTableSql = ` + CREATE TABLE revisions ( + id TEXT PRIMARY KEY, + parent_id TEXT NOT NULL DEFAULT "", + item_type INT NOT NULL, + item_id TEXT NOT NULL, + item_updated_time INT NOT NULL, + title_diff TEXT NOT NULL DEFAULT "", + body_diff TEXT NOT NULL DEFAULT "", + metadata_diff TEXT NOT NULL DEFAULT "", + encryption_cipher_text TEXT NOT NULL DEFAULT "", + encryption_applied INT NOT NULL DEFAULT 0, + updated_time INT NOT NULL, + created_time INT NOT NULL + ); + `; + queries.push(this.sqlStringToLines(newTableSql)[0]); + + queries.push('CREATE INDEX revisions_parent_id ON revisions (parent_id)'); + queries.push('CREATE INDEX revisions_item_type ON revisions (item_type)'); + queries.push('CREATE INDEX revisions_item_id ON revisions (item_id)'); + queries.push('CREATE INDEX revisions_item_updated_time ON revisions (item_updated_time)'); + queries.push('CREATE INDEX revisions_updated_time ON revisions (updated_time)'); + + queries.push('ALTER TABLE item_changes ADD COLUMN source INT NOT NULL DEFAULT 1'); + queries.push('ALTER TABLE item_changes ADD COLUMN before_change_item TEXT NOT NULL DEFAULT ""'); + } + queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] }); try { diff --git a/ReactNativeClient/lib/models/BaseItem.js b/ReactNativeClient/lib/models/BaseItem.js index 15c241289..d87588e57 100644 --- a/ReactNativeClient/lib/models/BaseItem.js +++ b/ReactNativeClient/lib/models/BaseItem.js @@ -160,6 +160,7 @@ class BaseItem extends BaseModel { } static async batchDelete(ids, options = null) { + if (!options) options = {}; let trackDeleted = true; if (options && options.trackDeleted !== null && options.trackDeleted !== undefined) trackDeleted = options.trackDeleted; @@ -219,6 +220,9 @@ class BaseItem extends BaseModel { if (['created_time', 'updated_time', 'sync_time', 'user_updated_time', 'user_created_time'].indexOf(propName) >= 0) { if (!propValue) return ''; propValue = moment.unix(propValue / 1000).utc().format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z'; + } else if (['title_diff', 'body_diff'].indexOf(propName) >= 0) { + if (!propValue) return ''; + propValue = JSON.stringify(propValue); } else if (propValue === null || propValue === undefined) { propValue = ''; } @@ -234,6 +238,9 @@ class BaseItem extends BaseModel { if (['created_time', 'updated_time', 'user_created_time', 'user_updated_time'].indexOf(propName) >= 0) { if (!propValue) return 0; propValue = moment(propValue, 'YYYY-MM-DDTHH:mm:ss.SSSZ').format('x'); + } else if (['title_diff', 'body_diff'].indexOf(propName) >= 0) { + if (!propValue) return ''; + propValue = JSON.parse(propValue); } else { propValue = Database.formatValue(ItemClass.fieldType(propName), propValue); } @@ -291,19 +298,16 @@ class BaseItem extends BaseModel { return this.encryptionService_; } + static revisionService() { + if (!this.revisionService_) throw new Error('BaseItem.revisionService_ is not set!!'); + return this.revisionService_; + } + static async serializeForSync(item) { const ItemClass = this.itemClass(item); let shownKeys = ItemClass.fieldNames(); shownKeys.push('type_'); - // if (ItemClass.syncExcludedKeys) { - // const keys = ItemClass.syncExcludedKeys(); - // for (let i = 0; i < keys.length; i++) { - // const idx = shownKeys.indexOf(keys[i]); - // shownKeys.splice(idx, 1); - // } - // } - const serialized = await ItemClass.serialize(item, shownKeys); if (!Setting.value('encryption.enabled') || !ItemClass.encryptionSupported()) { @@ -372,7 +376,7 @@ class BaseItem extends BaseModel { body.splice(0, 0, line); } } - + if (!output.type_) throw new Error('Missing required property: type_: ' + content); output.type_ = Number(output.type_); @@ -597,7 +601,7 @@ class BaseItem extends BaseModel { static updateSyncTimeQueries(syncTarget, item, syncTime, syncDisabled = false, syncDisabledReason = '') { const itemType = item.type_; const itemId = item.id; - if (!itemType || !itemId || syncTime === undefined) throw new Error('Invalid parameters in updateSyncTimeQueries()'); + if (!itemType || !itemId || syncTime === undefined) throw new Error(sprintf('Invalid parameters in updateSyncTimeQueries(): %d, %s, %d', syncTarget, JSON.stringify(item), syncTime)); return [ { @@ -704,6 +708,7 @@ class BaseItem extends BaseModel { } BaseItem.encryptionService_ = null; +BaseItem.revisionService_ = null; // Also update: // - itemsThatNeedSync() @@ -716,6 +721,7 @@ BaseItem.syncItemDefinitions_ = [ { type: BaseModel.TYPE_TAG, className: 'Tag' }, { type: BaseModel.TYPE_NOTE_TAG, className: 'NoteTag' }, { type: BaseModel.TYPE_MASTER_KEY, className: 'MasterKey' }, + { type: BaseModel.TYPE_REVISION, className: 'Revision' }, ]; module.exports = BaseItem; \ No newline at end of file diff --git a/ReactNativeClient/lib/models/ItemChange.js b/ReactNativeClient/lib/models/ItemChange.js index 4e23d628e..656a1dacd 100644 --- a/ReactNativeClient/lib/models/ItemChange.js +++ b/ReactNativeClient/lib/models/ItemChange.js @@ -11,7 +11,10 @@ class ItemChange extends BaseModel { return BaseModel.TYPE_ITEM_CHANGE; } - static async add(itemType, itemId, type) { + static async add(itemType, itemId, type, changeSource = null, beforeChangeItemJson = null) { + if (changeSource === null) changeSource = ItemChange.SOURCE_UNSPECIFIED; + if (!beforeChangeItemJson) beforeChangeItemJson = ''; + ItemChange.saveCalls_.push(true); // Using a mutex so that records can be added to the database in the @@ -21,7 +24,7 @@ class ItemChange extends BaseModel { try { await this.db().transactionExecBatch([ { sql: 'DELETE FROM item_changes WHERE item_id = ?', params: [itemId] }, - { sql: 'INSERT INTO item_changes (item_type, item_id, type, created_time) VALUES (?, ?, ?, ?)', params: [itemType, itemId, type, Date.now()] }, + { sql: 'INSERT INTO item_changes (item_type, item_id, type, source, created_time, before_change_item) VALUES (?, ?, ?, ?, ?, ?)', params: [itemType, itemId, type, changeSource, Date.now(), beforeChangeItemJson] }, ]); } finally { release(); @@ -61,4 +64,7 @@ ItemChange.TYPE_CREATE = 1; ItemChange.TYPE_UPDATE = 2; ItemChange.TYPE_DELETE = 3; +ItemChange.SOURCE_UNSPECIFIED = 1; +ItemChange.SOURCE_SYNC = 2; + module.exports = ItemChange; \ No newline at end of file diff --git a/ReactNativeClient/lib/models/Note.js b/ReactNativeClient/lib/models/Note.js index 7a006dd8a..9a283b93c 100644 --- a/ReactNativeClient/lib/models/Note.js +++ b/ReactNativeClient/lib/models/Note.js @@ -525,14 +525,32 @@ class Note extends BaseItem { return this.save(newNote); } + static async noteIsOlderThan(noteId, date) { + const n = await this.db().selectOne('SELECT updated_time FROM notes WHERE id = ?', [noteId]); + if (!n) throw new Error('No such note: ' + noteId); + return n.updated_time < date; + } + static async save(o, options = null) { let isNew = this.isNew(o, options); if (isNew && !o.source) o.source = Setting.value('appName'); if (isNew && !o.source_application) o.source_application = Setting.value('appId'); + // We only keep the previous note content for "old notes" (see Revision Service for more info) + // In theory, we could simply save all the previous note contents, and let the revision service + // decide what to keep and what to ignore, but in practice keeping the previous content is a bit + // heavy - the note needs to be reloaded here, the JSON blob needs to be saved, etc. + // So the check for old note here is basically an optimisation. + let beforeNoteJson = null; + if (!isNew && this.revisionService().isOldNote(o.id)) { + beforeNoteJson = await Note.load(o.id); + if (beforeNoteJson) beforeNoteJson = JSON.stringify(beforeNoteJson); + } + const note = await super.save(o, options); - ItemChange.add(BaseModel.TYPE_NOTE, note.id, isNew ? ItemChange.TYPE_CREATE : ItemChange.TYPE_UPDATE); + const changeSource = options && options.changeSource ? options.changeSource : null; + ItemChange.add(BaseModel.TYPE_NOTE, note.id, isNew ? ItemChange.TYPE_CREATE : ItemChange.TYPE_UPDATE, changeSource, beforeNoteJson); this.dispatch({ type: 'NOTE_UPDATE_ONE', @@ -550,16 +568,29 @@ class Note extends BaseItem { } static async batchDelete(ids, options = null) { - const result = await super.batchDelete(ids, options); - for (let i = 0; i < ids.length; i++) { - ItemChange.add(BaseModel.TYPE_NOTE, ids[i], ItemChange.TYPE_DELETE); + ids = ids.slice(); - this.dispatch({ - type: 'NOTE_DELETE', - id: ids[i], - }); + while (ids.length) { + const processIds = ids.splice(0, 50); + + const notes = await Note.byIds(processIds); + const beforeChangeItems = {}; + for (const note of notes) { + beforeChangeItems[note.id] = JSON.stringify(note); + } + + const result = await super.batchDelete(processIds, options); + const changeSource = options && options.changeSource ? options.changeSource : null; + for (let i = 0; i < processIds.length; i++) { + const id = processIds[i]; + ItemChange.add(BaseModel.TYPE_NOTE, id, ItemChange.TYPE_DELETE, changeSource, beforeChangeItems[id]); + + this.dispatch({ + type: 'NOTE_DELETE', + id: id, + }); + } } - return result; } static dueNotes() { diff --git a/ReactNativeClient/lib/models/Revision.js b/ReactNativeClient/lib/models/Revision.js new file mode 100644 index 000000000..e65e905cb --- /dev/null +++ b/ReactNativeClient/lib/models/Revision.js @@ -0,0 +1,248 @@ +const BaseModel = require('lib/BaseModel.js'); +const BaseItem = require('lib/models/BaseItem.js'); +const NoteTag = require('lib/models/NoteTag.js'); +const Note = require('lib/models/Note.js'); +const { time } = require('lib/time-utils.js'); +const { _ } = require('lib/locale'); +const DiffMatchPatch = require('diff-match-patch'); +const ArrayUtils = require('lib/ArrayUtils.js'); +const JoplinError = require('lib/JoplinError'); +const { sprintf } = require('sprintf-js'); + +const dmp = new DiffMatchPatch(); + +class Revision extends BaseItem { + + static tableName() { + return 'revisions'; + } + + static modelType() { + return BaseModel.TYPE_REVISION; + } + + static createTextPatch(oldText, newText) { + return dmp.patch_toText(dmp.patch_make(oldText, newText)); + } + + static applyTextPatch(text, patch) { + patch = dmp.patch_fromText(patch); + const result = dmp.patch_apply(patch, text); + if (!result || !result.length) throw new Error('Could not apply patch'); + return result[0]; + } + + static createObjectPatch(oldObject, newObject) { + if (!oldObject) oldObject = {}; + + const output = { + new: {}, + deleted: [], + }; + + for (let k in newObject) { + if (!newObject.hasOwnProperty(k)) continue; + if (oldObject[k] === newObject[k]) continue; + output.new[k] = newObject[k]; + } + + for (let k in oldObject) { + if (!oldObject.hasOwnProperty(k)) continue; + if (!(k in newObject)) output.deleted.push(k); + } + + return JSON.stringify(output); + } + + static applyObjectPatch(object, patch) { + patch = JSON.parse(patch); + const output = Object.assign({}, object); + + for (let k in patch.new) { + output[k] = patch.new[k]; + } + + for (let i = 0; i < patch.deleted.length; i++) { + delete output[patch.deleted[i]]; + } + + return output; + } + + static async countRevisions(itemType, itemId) { + const r = await this.db().selectOne('SELECT count(*) as total FROM revisions WHERE item_type = ? AND item_id = ?', [ + itemType, + itemId, + ]); + + return r ? r.total : 0; + } + + static latestRevision(itemType, itemId) { + return this.modelSelectOne('SELECT * FROM revisions WHERE item_type = ? AND item_id = ? ORDER BY item_updated_time DESC LIMIT 1', [ + itemType, + itemId, + ]); + } + + static allByType(itemType, itemId) { + return this.modelSelectAll('SELECT * FROM revisions WHERE item_type = ? AND item_id = ? ORDER BY item_updated_time ASC', [ + itemType, + itemId, + ]); + } + + static async itemsWithRevisions(itemType, itemIds) { + if (!itemIds.length) return []; + const rows = await this.db().selectAll('SELECT distinct item_id FROM revisions WHERE item_type = ? AND item_id IN ("' + itemIds.join('","') + '")', [ + itemType, + ]); + + return rows.map(r => r.item_id); + } + + static async itemsWithNoRevisions(itemType, itemIds) { + const withRevs = await this.itemsWithRevisions(itemType, itemIds); + const output = []; + for (let i = 0; i < itemIds.length; i++) { + if (withRevs.indexOf(itemIds[i]) < 0) output.push(itemIds[i]); + } + return ArrayUtils.unique(output); + } + + static moveRevisionToTop(revision, revs) { + let targetIndex = -1; + for (let i = revs.length - 1; i >= 0; i--) { + const rev = revs[i]; + if (rev.id === revision.id) { + targetIndex = i; + break; + } + } + + if (targetIndex < 0) throw new Error('Could not find revision: ' + revision.id); + + if (targetIndex !== revs.length - 1) { + revs = revs.slice(); + const toTop = revs[targetIndex]; + revs.splice(targetIndex, 1); + revs.push(toTop); + } + + return revs; + } + + // Note: revs must be sorted by update_time ASC (as returned by allByType) + static async mergeDiffs(revision, revs = null) { + if (!('encryption_applied' in revision) || !!revision.encryption_applied) throw new JoplinError('Target revision is encrypted', 'revision_encrypted'); + + if (!revs) { + revs = await this.modelSelectAll('SELECT * FROM revisions WHERE item_type = ? AND item_id = ? AND item_updated_time <= ? ORDER BY item_updated_time ASC', [ + revision.item_type, + revision.item_id, + revision.item_updated_time, + ]); + } else { + revs = revs.slice(); + } + + // Handle rare case where two revisions have been created at exactly the same millisecond + // Also handle even rarer case where a rev and its parent have been created at the + // same milliseconds. All code below expects target revision to be on top. + revs = this.moveRevisionToTop(revision, revs); + + const output = { + title: '', + body: '', + metadata: {}, + }; + + // Build up the list of revisions that are parents of the target revision. + const revIndexes = [revs.length - 1]; + let parentId = revision.parent_id; + for (let i = revs.length - 2; i >= 0; i--) { + const rev = revs[i]; + if (rev.id !== parentId) continue; + parentId = rev.parent_id; + revIndexes.push(i); + } + revIndexes.reverse(); + + for (const revIndex of revIndexes) { + const rev = revs[revIndex]; + if (!!rev.encryption_applied) throw new JoplinError(sprintf('Revision "%s" is encrypted', rev.id), 'revision_encrypted'); + output.title = this.applyTextPatch(output.title, rev.title_diff); + output.body = this.applyTextPatch(output.body, rev.body_diff); + output.metadata = this.applyObjectPatch(output.metadata, rev.metadata_diff); + } + + return output; + } + + static async deleteOldRevisions(ttl) { + // When deleting old revisions, we need to make sure that the oldest surviving revision + // is a "merged" one (as opposed to a diff from a now deleted revision). So every time + // we deleted a revision, we need to find if there's a corresponding surviving revision + // and modify that revision into a "merged" one. + + const cutOffDate = Date.now() - ttl; + const revisions = await this.modelSelectAll('SELECT * FROM revisions WHERE item_updated_time < ? ORDER BY item_updated_time DESC', [cutOffDate]); + const doneItems = {}; + + for (const rev of revisions) { + const doneKey = rev.item_type + '_' + rev.item_id; + if (doneItems[doneKey]) continue; + + const keptRev = await this.modelSelectOne('SELECT * FROM revisions WHERE item_updated_time >= ? AND item_type = ? AND item_id = ? ORDER BY item_updated_time ASC LIMIT 1', [ + cutOffDate, + rev.item_type, + rev.item_id, + ]); + + try { + const deleteQueryCondition = 'item_updated_time < ? AND item_id = ?'; + const deleteQueryParams = [cutOffDate, rev.item_id]; + const deleteQuery = { sql: 'DELETE FROM revisions WHERE ' + deleteQueryCondition, params: deleteQueryParams }; + + if (!keptRev) { + const hasEncrypted = await this.modelSelectOne('SELECT * FROM revisions WHERE encryption_applied = 1 AND ' + deleteQueryCondition, deleteQueryParams); + if (!!hasEncrypted) throw new JoplinError('One of the revision to be deleted is encrypted', 'revision_encrypted'); + await this.db().transactionExecBatch([deleteQuery]); + } else { + // Note: we don't need to check for encrypted rev here because + // mergeDiff will already throw the revision_encrypted exception + // if a rev is encrypted. + const merged = await this.mergeDiffs(keptRev); + + const queries = [ + deleteQuery, + { sql: 'UPDATE revisions SET title_diff = ?, body_diff = ?, metadata_diff = ? WHERE id = ?', params: [ + this.createTextPatch('', merged.title), + this.createTextPatch('', merged.body), + this.createObjectPatch({}, merged.metadata), + keptRev.id, + ] }, + ]; + + await this.db().transactionExecBatch(queries); + } + } catch (error) { + if (error.code === 'revision_encrypted') { + this.logger().info('Aborted deletion of old revisions for item ' + rev.item_id + ' because one of the revisions is still encrypted', error); + } else { + throw error; + } + } + + doneItems[doneKey] = true; + } + } + + static async revisionExists(itemType, itemId, updatedTime) { + const existingRev = await Revision.latestRevision(itemType, itemId); + return existingRev && existingRev.item_updated_time === updatedTime; + } + +} + +module.exports = Revision; diff --git a/ReactNativeClient/lib/models/Setting.js b/ReactNativeClient/lib/models/Setting.js index 82c9e9718..4d3e1d92a 100644 --- a/ReactNativeClient/lib/models/Setting.js +++ b/ReactNativeClient/lib/models/Setting.js @@ -191,7 +191,14 @@ class Setting extends BaseModel { 'resourceService.lastProcessedChangeId': { value: 0, type: Setting.TYPE_INT, public: false }, 'searchEngine.lastProcessedChangeId': { value: 0, type: Setting.TYPE_INT, public: false }, + 'revisionService.lastProcessedChangeId': { value: 0, type: Setting.TYPE_INT, public: false }, + 'searchEngine.initialIndexingDone': { value: false, type: Setting.TYPE_BOOL, public: false }, + + 'revisionService.enabled': { section: 'revisionService', value: true, type: Setting.TYPE_BOOL, public: true, label: () => _('Enable note history') }, + 'revisionService.ttlDays': { section: 'revisionService', value: 90, type: Setting.TYPE_INT, public: true, minimum: 1, maximum: 365 * 2, step: 1, unitLabel: (value = null) => { return value === null ? _('days') : _('%d days', value) }, label: () => _('Keep note history for') }, + 'revisionService.installedTime': { section: 'revisionService', value: 0, type: Setting.TYPE_INT, public: false }, + 'welcome.wasBuilt': { value: false, type: Setting.TYPE_BOOL, public: false }, }; @@ -586,6 +593,7 @@ class Setting extends BaseModel { if (name === 'note') return _('Note'); if (name === 'plugins') return _('Plugins'); if (name === 'application') return _('Application'); + if (name === 'revisionService') return _('Note History'); return name; } diff --git a/ReactNativeClient/lib/services/ExternalEditWatcher.js b/ReactNativeClient/lib/services/ExternalEditWatcher.js index ade123444..d3463dfa3 100644 --- a/ReactNativeClient/lib/services/ExternalEditWatcher.js +++ b/ReactNativeClient/lib/services/ExternalEditWatcher.js @@ -7,6 +7,7 @@ const { splitCommandString } = require('lib/string-utils'); const { fileExtension } = require('lib/path-utils'); const spawn = require('child_process').spawn; const chokidar = require('chokidar'); +// const chokidar = null; class ExternalEditWatcher { diff --git a/ReactNativeClient/lib/services/ItemChangeUtils.js b/ReactNativeClient/lib/services/ItemChangeUtils.js index 2debc7fd1..ed34ab74d 100644 --- a/ReactNativeClient/lib/services/ItemChangeUtils.js +++ b/ReactNativeClient/lib/services/ItemChangeUtils.js @@ -7,6 +7,7 @@ class ItemChangeUtils { const lastProcessedChangeIds = [ Setting.value('resourceService.lastProcessedChangeId'), Setting.value('searchEngine.lastProcessedChangeId'), + Setting.value('revisionService.lastProcessedChangeId'), ]; const lowestChangeId = Math.min(...lastProcessedChangeIds); diff --git a/ReactNativeClient/lib/services/ResourceService.js b/ReactNativeClient/lib/services/ResourceService.js index 97a2c2c99..153156296 100644 --- a/ReactNativeClient/lib/services/ResourceService.js +++ b/ReactNativeClient/lib/services/ResourceService.js @@ -93,6 +93,7 @@ class ResourceService extends BaseService { } async deleteOrphanResources(expiryDelay = null) { + if (expiryDelay === null) expiryDelay = Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000; const resourceIds = await NoteResource.orphanResources(expiryDelay); this.logger().info('ResourceService::deleteOrphanResources:', resourceIds); for (let i = 0; i < resourceIds.length; i++) { diff --git a/ReactNativeClient/lib/services/RevisionService.js b/ReactNativeClient/lib/services/RevisionService.js new file mode 100644 index 000000000..0c48f27f4 --- /dev/null +++ b/ReactNativeClient/lib/services/RevisionService.js @@ -0,0 +1,277 @@ +const { Logger } = require('lib/logger.js'); +const ItemChange = require('lib/models/ItemChange'); +const Note = require('lib/models/Note'); +const Folder = require('lib/models/Folder'); +const Setting = require('lib/models/Setting'); +const Revision = require('lib/models/Revision'); +const BaseModel = require('lib/BaseModel'); +const ItemChangeUtils = require('lib/services/ItemChangeUtils'); +const { shim } = require('lib/shim'); +const BaseService = require('lib/services/BaseService'); +const { _ } = require('lib/locale.js'); +const ArrayUtils = require('lib/ArrayUtils.js'); + +class RevisionService extends BaseService { + + constructor() { + super(); + + // An "old note" is one that has been created before the revision service existed. These + // notes never benefited from revisions so the first time they are modified, a copy of + // the original note is saved. The goal is to have at least one revision in case the note + // is deleted or modified as a result of a bug or user mistake. + this.isOldNotesCache_ = {}; + + if (!Setting.value('revisionService.installedTime')) Setting.setValue('revisionService.installedTime', Date.now()); + } + + installedTime() { + return Setting.value('revisionService.installedTime'); + } + + static instance() { + if (this.instance_) return this.instance_; + this.instance_ = new RevisionService(); + return this.instance_; + } + + async isOldNote(noteId) { + if (noteId in this.isOldNotesCache_) return this.isOldNotesCache_[noteId]; + + const r = await Note.noteIsOlderThan(noteId, this.installedTime()); + this.isOldNotesCache_[noteId] = r; + return r; + } + + noteMetadata_(note) { + const excludedFields = ['type_', 'title', 'body', 'created_time', 'updated_time', 'encryption_applied', 'encryption_cipher_text', 'is_conflict']; + const md = {}; + for (let k in note) { + if (excludedFields.indexOf(k) >= 0) continue; + md[k] = note[k]; + } + return md; + } + + async createNoteRevision(note, parentRevId = null) { + const parentRev = parentRevId ? await Revision.load(parentRevId) : await Revision.latestRevision(BaseModel.TYPE_NOTE, note.id); + + const output = { + parent_id: '', + item_type: BaseModel.TYPE_NOTE, + item_id: note.id, + item_updated_time: note.updated_time, + }; + + const noteMd = this.noteMetadata_(note); + const noteTitle = note.title ? note.title : ''; + const noteBody = note.body ? note.body : ''; + + if (!parentRev) { + output.title_diff = Revision.createTextPatch('', noteTitle); + output.body_diff = Revision.createTextPatch('', noteBody); + output.metadata_diff = Revision.createObjectPatch({}, noteMd); + } else { + const merged = await Revision.mergeDiffs(parentRev); + output.parent_id = parentRev.id; + output.title_diff = Revision.createTextPatch(merged.title, noteTitle); + output.body_diff = Revision.createTextPatch(merged.body, noteBody); + output.metadata_diff = Revision.createObjectPatch(merged.metadata, noteMd); + } + + return Revision.save(output); + } + + async createNoteRevisionsByIds(noteIds) { + noteIds = ArrayUtils.unique(noteIds); + + while (noteIds.length) { + const ids = noteIds.splice(0, 100); + const notes = await Note.byIds(ids); + for (const note of notes) { + const existingRev = await Revision.latestRevision(BaseModel.TYPE_NOTE, note.id); + if (existingRev && existingRev.item_updated_time === note.updated_time) continue; + await this.createNoteRevision(note); + } + } + } + + async createNoteRevisionIfNoneFound(noteId, cutOffDate) { + const count = await Revision.countRevisions(BaseModel.TYPE_NOTE, noteId); + if (count) return; + + const note = await Note.load(noteId); + if (!note) { + this.logger().warn('RevisionService:createNoteRevisionIfNoneFound: Could not find note ' + noteId); + return; + } + + if (note.updated_time > cutOffDate) return; + + await this.createNoteRevision(note); + } + + async collectRevisions() { + if (this.isCollecting_) return; + + this.isCollecting_ = true; + + await ItemChange.waitForAllSaved(); + + const doneNoteIds = []; + + try { + while (true) { + // See synchronizer test units to see why changes coming + // from sync are skipped. + const changes = await ItemChange.modelSelectAll(` + SELECT id, item_id, type, before_change_item + FROM item_changes + WHERE item_type = ? + AND source != ? + AND id > ? + ORDER BY id ASC + LIMIT 10 + `, [BaseModel.TYPE_NOTE, ItemChange.SOURCE_SYNC, Setting.value('revisionService.lastProcessedChangeId')]); + + if (!changes.length) break; + + const noteIds = changes.map(a => a.item_id); + const notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND encryption_applied = 0 AND id IN ("' + noteIds.join('","') + '")'); + + for (let i = 0; i < changes.length; i++) { + const change = changes[i]; + const noteId = change.item_id; + + if (change.type === ItemChange.TYPE_UPDATE && doneNoteIds.indexOf(noteId) < 0) { + const note = BaseModel.byId(notes, noteId); + const oldNote = change.before_change_item ? JSON.parse(change.before_change_item) : null; + + if (note) { + if (oldNote && oldNote.updated_time < this.installedTime()) { + // This is where we save the original version of this old note + await this.createNoteRevision(oldNote); + } + + await this.createNoteRevision(note); + doneNoteIds.push(noteId); + this.isOldNotesCache_[noteId] = false; + } + } + + if (change.type === ItemChange.TYPE_DELETE && !!change.before_change_item) { + const note = JSON.parse(change.before_change_item); + const revExists = await Revision.revisionExists(BaseModel.TYPE_NOTE, note.id, note.updated_time); + if (!revExists) await this.createNoteRevision(note); + doneNoteIds.push(noteId); + } + + Setting.setValue('revisionService.lastProcessedChangeId', change.id); + } + } + } catch (error) { + if (error.code === 'revision_encrypted') { + // One or more revisions are encrypted - stop processing for now + // and these revisions will be processed next time the revision + // collector runs. + this.logger().info('RevisionService::collectRevisions: One or more revision was encrypted. Processing was stopped but will resume later when the revision is decrypted.', error); + } else { + this.logger().error('RevisionService::collectRevisions:', error); + } + } + + await Setting.saveAll(); + await ItemChangeUtils.deleteProcessedChanges(); + + this.isCollecting_ = false; + + this.logger().info('RevisionService::collectRevisions: Created revisions for ' + doneNoteIds.length + ' notes'); + } + + async deleteOldRevisions(ttl) { + return Revision.deleteOldRevisions(ttl); + } + + async revisionNote(revisions, index) { + if (index < 0 || index >= revisions.length) throw new Error('Invalid revision index: ' + index); + + const rev = revisions[index]; + const merged = await Revision.mergeDiffs(rev, revisions); + + const output = Object.assign({ + title: merged.title, + body: merged.body, + }, merged.metadata); + output.updated_time = output.user_updated_time; + output.created_time = output.user_created_time; + output.type_ = BaseModel.TYPE_NOTE; + + return output; + } + + restoreFolderTitle() { + return _('Restored Notes'); + } + + async restoreFolder() { + let folder = await Folder.loadByTitle(this.restoreFolderTitle()); + if (!folder) { + folder = await Folder.save({ title: this.restoreFolderTitle() }); + } + return folder; + } + + async importRevisionNote(note) { + const toImport = Object.assign({}, note); + delete toImport.id; + delete toImport.updated_time; + delete toImport.created_time; + delete toImport.encryption_applied; + delete toImport.encryption_cipher_text; + + const folder = await this.restoreFolder(); + + toImport.parent_id = folder.id; + + await Note.save(toImport); + } + + async maintenance() { + const startTime = Date.now(); + this.logger().info('RevisionService::maintenance: Starting...'); + + if (!Setting.value('revisionService.enabled')) { + this.logger().info('RevisionService::maintenance: Service is disabled'); + // We do as if we had processed all the latest changes so that they can be cleaned up + // later on by ItemChangeUtils.deleteProcessedChanges(). + Setting.setValue('revisionService.lastProcessedChangeId', await ItemChange.lastChangeId()); + await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000); + } else { + this.logger().info('RevisionService::maintenance: Service is enabled'); + await this.collectRevisions(); + await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000); + } + + this.logger().info('RevisionService::maintenance: Done in ' + (Date.now() - startTime) + 'ms'); + } + + runInBackground(collectRevisionInterval = null) { + if (this.isRunningInBackground_) return; + this.isRunningInBackground_ = true; + + if (collectRevisionInterval === null) collectRevisionInterval = 1000 * 60 * 10; + + this.logger().info('RevisionService::runInBackground: Starting background service with revision collection interval ' + collectRevisionInterval); + + setTimeout(() => { + this.maintenance(); + }, 1000 * 4); + + shim.setInterval(() => { + this.maintenance(); + }, collectRevisionInterval); + } + +} + +module.exports = RevisionService; \ No newline at end of file diff --git a/ReactNativeClient/lib/synchronizer.js b/ReactNativeClient/lib/synchronizer.js index eb2a16a51..a567723af 100644 --- a/ReactNativeClient/lib/synchronizer.js +++ b/ReactNativeClient/lib/synchronizer.js @@ -2,6 +2,7 @@ const BaseItem = require('lib/models/BaseItem.js'); const Folder = require('lib/models/Folder.js'); const Note = require('lib/models/Note.js'); const Resource = require('lib/models/Resource.js'); +const ItemChange = require('lib/models/ItemChange.js'); const ResourceLocalState = require('lib/models/ResourceLocalState.js'); const MasterKey = require('lib/models/MasterKey.js'); const BaseModel = require('lib/BaseModel.js'); @@ -325,7 +326,7 @@ class Synchronizer { if (action == "createRemote" || action == "updateRemote") { let canSync = true; try { - if (this.testingHooks_.indexOf("rejectedByTarget") >= 0) throw new JoplinError("Testing rejectedByTarget", "rejectedByTarget"); + if (this.testingHooks_.indexOf("notesRejectedByTarget") >= 0 && local.type_ === BaseModel.TYPE_NOTE) throw new JoplinError("Testing rejectedByTarget", "rejectedByTarget"); const content = await ItemClass.serializeForSync(local); await this.api().put(path, content); } catch (error) { @@ -370,9 +371,9 @@ class Synchronizer { local = remoteContent; const syncTimeQueries = BaseItem.updateSyncTimeQueries(syncTargetId, local, time.unixMs()); - await ItemClass.save(local, { autoTimestamp: false, nextQueries: syncTimeQueries }); + await ItemClass.save(local, { autoTimestamp: false, changeSource: ItemChange.SOURCE_SYNC, nextQueries: syncTimeQueries }); } else { - await ItemClass.delete(local.id); + await ItemClass.delete(local.id, { changeSource: ItemChange.SOURCE_SYNC }); } } else if (action == "noteConflict") { // ------------------------------------------------------------------------------ @@ -395,7 +396,7 @@ class Synchronizer { let conflictedNote = Object.assign({}, local); delete conflictedNote.id; conflictedNote.is_conflict = 1; - await Note.save(conflictedNote, { autoTimestamp: false }); + await Note.save(conflictedNote, { autoTimestamp: false, changeSource: ItemChange.SOURCE_SYNC }); } // ------------------------------------------------------------------------------ @@ -406,12 +407,12 @@ class Synchronizer { if (remote) { local = remoteContent; const syncTimeQueries = BaseItem.updateSyncTimeQueries(syncTargetId, local, time.unixMs()); - await ItemClass.save(local, { autoTimestamp: false, nextQueries: syncTimeQueries }); + await ItemClass.save(local, { autoTimestamp: false, changeSource: ItemChange.SOURCE_SYNC, nextQueries: syncTimeQueries }); if (!!local.encryption_applied) this.dispatch({ type: "SYNC_GOT_ENCRYPTED_ITEM" }); } else { // Remote no longer exists (note deleted) so delete local one too - await ItemClass.delete(local.id); + await ItemClass.delete(local.id, { changeSource: ItemChange.SOURCE_SYNC }); } } @@ -535,6 +536,8 @@ class Synchronizer { } } + if (this.testingHooks_.indexOf('skipRevisions') >= 0 && content && content.type_ === BaseModel.TYPE_REVISION) action = null; + if (!action) continue; this.logSyncOperation(action, local, remote, reason); @@ -557,30 +560,13 @@ class Synchronizer { let options = { autoTimestamp: false, nextQueries: BaseItem.updateSyncTimeQueries(syncTargetId, content, time.unixMs()), + changeSource: ItemChange.SOURCE_SYNC, }; if (action == "createLocal") options.isNew = true; if (action == "updateLocal") options.oldItem = local; const creatingNewResource = content.type_ == BaseModel.TYPE_RESOURCE && action == "createLocal"; - // if (content.type_ == BaseModel.TYPE_RESOURCE && action == "createLocal") { - // let localResourceContentPath = Resource.fullPath(content); - // let remoteResourceContentPath = this.resourceDirName_ + "/" + content.id; - // try { - // await this.api().get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" }); - // } catch (error) { - // if (error.code === 'rejectedByTarget') { - // this.progressReport_.errors.push(error); - // this.logger().warn('Rejected by target: ' + path + ': ' + error.message); - // continue; - // } else { - // throw error; - // } - // } - // } - - // if (creatingNewResource) content.fetch_status = Resource.FETCH_STATUS_IDLE; - if (creatingNewResource) { await ResourceLocalState.save({ resource_id: content.id, fetch_status: Resource.FETCH_STATUS_IDLE }); } @@ -608,7 +594,7 @@ class Synchronizer { } let ItemClass = BaseItem.itemClass(local.type_); - await ItemClass.delete(local.id, { trackDeleted: false }); + await ItemClass.delete(local.id, { trackDeleted: false, changeSource: ItemChange.SOURCE_SYNC }); } } @@ -653,7 +639,7 @@ class Synchronizer { // CONFLICT await Folder.markNotesAsConflict(item.id); } - await Folder.delete(item.id, { deleteChildren: false, trackDeleted: false }); + await Folder.delete(item.id, { deleteChildren: false, changeSource: ItemChange.SOURCE_SYNC, trackDeleted: false }); } } diff --git a/ReactNativeClient/package-lock.json b/ReactNativeClient/package-lock.json index 5bfe9e46f..edb7a28b8 100644 --- a/ReactNativeClient/package-lock.json +++ b/ReactNativeClient/package-lock.json @@ -2325,6 +2325,11 @@ "resolved": "https://registry.npmjs.org/diacritics/-/diacritics-1.3.0.tgz", "integrity": "sha1-PvqHMj67hj5mls67AILUj/PW96E=" }, + "diff-match-patch": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.4.tgz", + "integrity": "sha512-Uv3SW8bmH9nAtHKaKSanOQmj2DnlH65fUpcrMdfdaOxUG02QQ4YGZ8AE7kKOMisF7UqvOlGKVYWRvezdncW9lg==" + }, "dom-walk": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.1.tgz", @@ -2680,7 +2685,8 @@ }, "ansi-regex": { "version": "2.1.1", - "bundled": true + "bundled": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -2698,11 +2704,13 @@ }, "balanced-match": { "version": "1.0.0", - "bundled": true + "bundled": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2715,15 +2723,18 @@ }, "code-point-at": { "version": "1.1.0", - "bundled": true + "bundled": true, + "optional": true }, "concat-map": { "version": "0.0.1", - "bundled": true + "bundled": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", - "bundled": true + "bundled": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -2826,7 +2837,8 @@ }, "inherits": { "version": "2.0.3", - "bundled": true + "bundled": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -2836,6 +2848,7 @@ "is-fullwidth-code-point": { "version": "1.0.0", "bundled": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -2848,17 +2861,20 @@ "minimatch": { "version": "3.0.4", "bundled": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { "version": "0.0.8", - "bundled": true + "bundled": true, + "optional": true }, "minipass": { "version": "2.2.4", "bundled": true, + "optional": true, "requires": { "safe-buffer": "^5.1.1", "yallist": "^3.0.0" @@ -2875,6 +2891,7 @@ "mkdirp": { "version": "0.5.1", "bundled": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -2947,7 +2964,8 @@ }, "number-is-nan": { "version": "1.0.1", - "bundled": true + "bundled": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -2957,6 +2975,7 @@ "once": { "version": "1.4.0", "bundled": true, + "optional": true, "requires": { "wrappy": "1" } @@ -3032,7 +3051,8 @@ }, "safe-buffer": { "version": "5.1.1", - "bundled": true + "bundled": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -3062,6 +3082,7 @@ "string-width": { "version": "1.0.2", "bundled": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -3079,6 +3100,7 @@ "strip-ansi": { "version": "3.0.1", "bundled": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -3117,11 +3139,13 @@ }, "wrappy": { "version": "1.0.2", - "bundled": true + "bundled": true, + "optional": true }, "yallist": { "version": "3.0.2", - "bundled": true + "bundled": true, + "optional": true } } }, @@ -4123,7 +4147,7 @@ }, "load-json-file": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", "requires": { "graceful-fs": "^4.1.2", @@ -5278,7 +5302,7 @@ }, "load-json-file": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", "requires": { "graceful-fs": "^4.1.2", @@ -7372,7 +7396,7 @@ "dependencies": { "uuid": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/uuid/-/uuid-3.0.1.tgz", "integrity": "sha1-ZUS7ot/ajBzxfmKaOjBeK7H+5sE=" } } diff --git a/ReactNativeClient/package.json b/ReactNativeClient/package.json index 2fd439b5d..ea666cf76 100644 --- a/ReactNativeClient/package.json +++ b/ReactNativeClient/package.json @@ -13,6 +13,7 @@ "base-64": "^0.1.0", "buffer": "^5.0.8", "diacritics": "^1.3.0", + "diff-match-patch": "^1.0.4", "events": "^1.1.1", "form-data": "^2.1.4", "highlight.js": "^9.15.6", diff --git a/ReactNativeClient/root.js b/ReactNativeClient/root.js index 63cb0aebf..d1e91cfea 100644 --- a/ReactNativeClient/root.js +++ b/ReactNativeClient/root.js @@ -22,9 +22,11 @@ const Tag = require('lib/models/Tag.js'); const NoteTag = require('lib/models/NoteTag.js'); const BaseItem = require('lib/models/BaseItem.js'); const MasterKey = require('lib/models/MasterKey.js'); +const Revision = require('lib/models/Revision.js'); const BaseModel = require('lib/BaseModel.js'); const BaseService = require('lib/services/BaseService.js'); const ResourceService = require('lib/services/ResourceService'); +const RevisionService = require('lib/services/RevisionService'); const { JoplinDatabase } = require('lib/joplin-database.js'); const { Database } = require('lib/database.js'); const { NotesScreen } = require('lib/components/screens/notes.js'); @@ -396,6 +398,7 @@ async function initialize(dispatch) { BaseItem.loadClass('Tag', Tag); BaseItem.loadClass('NoteTag', NoteTag); BaseItem.loadClass('MasterKey', MasterKey); + BaseItem.loadClass('Revision', Revision); const fsDriver = new FsDriverRN(); @@ -430,6 +433,8 @@ async function initialize(dispatch) { reg.logger().info('db.ftsEnabled = ', Setting.value('db.ftsEnabled')); } + BaseItem.revisionService_ = RevisionService.instance(); + // Note: for now we hard-code the folder sort order as we need to // create a UI to allow customisation (started in branch mobile_add_sidebar_buttons) Setting.setValue('folders.sortOrder.field', 'title'); @@ -526,6 +531,10 @@ async function initialize(dispatch) { await WelcomeUtils.install(dispatch); + // Collect revisions more frequently on mobile because it doesn't auto-save + // and it cannot collect anything when the app is not active. + RevisionService.instance().runInBackground(1000 * 30); + reg.logger().info('Application initialized'); }