From 8a6fe20a69e9347ac7cff6424f5c558d461594f5 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 22 May 2019 15:56:07 +0100 Subject: [PATCH] All: Resolves #1481: New: Allow downloading attachments on demand or automatically (#1527) * Allow downloading resources automatically, on demand, or when loading note * Make needToBeFetched calls to return the right number of resources * All: Improved handling of resource downloading and decryption * Desktop: Click on resource to download it (and, optionally, to decrypt it) * Desktop: Better handling of resource state (not downloaded, downloading, encrypted) in front end * Renamed setting to sync.resourceDownloadMode * Download resources when changing setting * tweaks * removed duplicate cs * Better report resource download progress * Make sure resource cache is properly cleared when needed * Also handle manual download for non-image resources * More improvements to logic when downloading and decrypting resources --- CliClient/tests/synchronizer.js | 44 ++++++++- ElectronClient/app/gui/NoteText.jsx | 28 ++++-- ElectronClient/app/gui/SideBar.jsx | 2 +- ElectronClient/app/gui/note-viewer/index.html | 6 +- ReactNativeClient/lib/BaseApplication.js | 9 +- ReactNativeClient/lib/MdToHtml/noteStyle.js | 15 +++ .../lib/MdToHtml/rules/html_image.js | 10 +- ReactNativeClient/lib/MdToHtml/rules/image.js | 10 +- .../lib/MdToHtml/rules/link_open.js | 17 +++- ReactNativeClient/lib/MdToHtml/utils.js | 97 ++++++++++++++++--- ReactNativeClient/lib/MdToHtml/webviewLib.js | 31 +++++- .../lib/components/note-body-viewer.js | 19 +++- .../lib/components/screens/note.js | 19 +++- .../components/shared/note-screen-shared.js | 34 ++++++- .../shared/reduxSharedMiddleware.js | 5 + .../lib/components/side-menu-content.js | 4 +- ReactNativeClient/lib/joplin-database.js | 17 +++- ReactNativeClient/lib/models/BaseItem.js | 8 +- ReactNativeClient/lib/models/Resource.js | 66 +++++++------ ReactNativeClient/lib/models/Setting.js | 8 ++ .../lib/services/DecryptionWorker.js | 40 ++++++-- .../lib/services/ResourceFetcher.js | 46 ++++++--- ReactNativeClient/root.js | 43 +++++++- 23 files changed, 470 insertions(+), 108 deletions(-) diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index 2c89bf3d0..ba205ae67 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -865,7 +865,7 @@ describe('Synchronizer', function() { expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_IDLE); const fetcher = new ResourceFetcher(() => { return synchronizer().api() }); - fetcher.queueDownload(resource1_2.id); + fetcher.queueDownload_(resource1_2.id); await fetcher.waitForAllFinished(); resource1_2 = await Resource.load(resource1.id); @@ -894,7 +894,7 @@ describe('Synchronizer', function() { // Simulate a failed download get: () => { return new Promise((resolve, reject) => { reject(new Error('did not work')) }); } } }); - fetcher.queueDownload(resource1.id); + fetcher.queueDownload_(resource1.id); await fetcher.waitForAllFinished(); resource1 = await Resource.load(resource1.id); @@ -920,7 +920,7 @@ describe('Synchronizer', function() { expect(r1.size).toBe(-1); const fetcher = new ResourceFetcher(() => { return synchronizer().api() }); - fetcher.queueDownload(r1.id); + fetcher.queueDownload_(r1.id); await fetcher.waitForAllFinished(); r1 = await Resource.load(r1.id); expect(r1.size).toBe(2720); @@ -974,7 +974,7 @@ describe('Synchronizer', function() { await encryptionService().loadMasterKeysFromSettings(); const fetcher = new ResourceFetcher(() => { return synchronizer().api() }); - fetcher.queueDownload(resource1.id); + fetcher.queueDownload_(resource1.id); await fetcher.waitForAllFinished(); let resource1_2 = (await Resource.all())[0]; @@ -1057,7 +1057,7 @@ describe('Synchronizer', function() { await encryptionService().loadMasterKeysFromSettings(); const fetcher = new ResourceFetcher(() => { return synchronizer().api() }); - fetcher.queueDownload(resource1.id); + fetcher.queueDownload_(resource1.id); await fetcher.waitForAllFinished(); await decryptionWorker().start(); @@ -1351,4 +1351,38 @@ describe('Synchronizer', function() { expect((await remoteResources()).length).toBe(1); })); + it('should decrypt the resource metadata, but not try to decrypt the file, if it is not present', asyncTest(async () => { + const note1 = await Note.save({ title: 'note' }); + await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg'); + const masterKey = await loadEncryptionMasterKey(); + await encryptionService().enableEncryption(masterKey, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + await synchronizer().start(); + expect(await allSyncTargetItemsEncrypted()).toBe(true); + + await switchClient(2); + + await synchronizer().start(); + Setting.setObjectKey('encryption.passwordCache', masterKey.id, '123456'); + await encryptionService().loadMasterKeysFromSettings(); + await decryptionWorker().start(); + + let resource = (await Resource.all())[0]; + + expect(!!resource.encryption_applied).toBe(false); + expect(!!resource.encryption_blob_encrypted).toBe(true); + + const resourceFetcher = new ResourceFetcher(() => { return synchronizer().api() }); + await resourceFetcher.start(); + await resourceFetcher.waitForAllFinished(); + + const ls = await Resource.localState(resource); + expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_DONE); + + await decryptionWorker().start(); + resource = (await Resource.all())[0]; + + expect(!!resource.encryption_blob_encrypted).toBe(false); + })); + }); diff --git a/ElectronClient/app/gui/NoteText.jsx b/ElectronClient/app/gui/NoteText.jsx index fa197ff68..7aab6eaa3 100644 --- a/ElectronClient/app/gui/NoteText.jsx +++ b/ElectronClient/app/gui/NoteText.jsx @@ -36,6 +36,7 @@ const ResourceFetcher = require('lib/services/ResourceFetcher'); const { toSystemSlashes, safeFilename } = require('lib/path-utils'); const { clipboard } = require('electron'); const SearchEngine = require('lib/services/SearchEngine'); +const DecryptionWorker = require('lib/services/DecryptionWorker'); const ModelCache = require('lib/services/ModelCache'); const NoteTextViewer = require('./NoteTextViewer.min'); const NoteRevisionViewer = require('./NoteRevisionViewer.min'); @@ -226,14 +227,13 @@ class NoteTextComponent extends React.Component { } } - this.resourceFetcher_downloadComplete = async (resource) => { + this.refreshResource = async (event) => { if (!this.state.note || !this.state.note.body) return; const resourceIds = await Note.linkedResourceIds(this.state.note.body); - if (resourceIds.indexOf(resource.id) >= 0) { - // this.mdToHtml().clearCache(); + if (resourceIds.indexOf(event.id) >= 0) { + shared.clearResourceCache(); this.lastSetHtml_ = ''; this.scheduleHtmlUpdate(); - //this.updateHtml(this.state.note.body); } } @@ -363,7 +363,8 @@ class NoteTextComponent extends React.Component { eventManager.on('noteTypeToggle', this.onNoteTypeToggle_); eventManager.on('todoToggle', this.onTodoToggle_); - ResourceFetcher.instance().on('downloadComplete', this.resourceFetcher_downloadComplete); + shared.installResourceHandling(this.refreshResource); + ExternalEditWatcher.instance().on('noteChange', this.externalEditWatcher_noteChange); } @@ -376,7 +377,8 @@ class NoteTextComponent extends React.Component { eventManager.removeListener('noteTypeToggle', this.onNoteTypeToggle_); eventManager.removeListener('todoToggle', this.onTodoToggle_); - ResourceFetcher.instance().off('downloadComplete', this.resourceFetcher_downloadComplete); + shared.uninstallResourceHandling(this.refreshResource); + ExternalEditWatcher.instance().off('noteChange', this.externalEditWatcher_noteChange); } @@ -474,6 +476,8 @@ class NoteTextComponent extends React.Component { // Scroll back to top when loading new note if (loadingNewNote) { + shared.clearResourceCache(); + this.editorMaxScrollTop_ = 0; // HACK: To go around a bug in Ace editor, we first set the scroll position to 1 @@ -521,6 +525,11 @@ class NoteTextComponent extends React.Component { this.setViewerPercentScroll(scrollPercent ? scrollPercent : 0); }, 10); } + + if (note && note.body && Setting.value('sync.resourceDownloadMode') === 'auto') { + const resourceIds = await Note.linkedResourceIds(note.body); + await ResourceFetcher.instance().markForDownload(resourceIds); + } } if (note) { @@ -647,7 +656,7 @@ class NoteTextComponent extends React.Component { async webview_ipcMessage(event) { const msg = event.channel ? event.channel : ''; - const args = event.args; + const args = event.args; const arg0 = args && args.length >= 1 ? args[0] : null; const arg1 = args && args.length >= 2 ? args[1] : null; @@ -666,6 +675,10 @@ class NoteTextComponent extends React.Component { const ls = Object.assign({}, this.state.localSearch); ls.resultCount = arg0; this.setState({ localSearch: ls }); + } else if (msg.indexOf('markForDownload:') === 0) { + const s = msg.split(':'); + if (s.length < 2) throw new Error('Invalid message: ' + msg); + ResourceFetcher.instance().markForDownload(s[1]); } else if (msg === 'percentScroll') { this.ignoreNextEditorScroll_ = true; this.setEditorPercentScroll(arg0); @@ -1741,6 +1754,7 @@ class NoteTextComponent extends React.Component { if (htmlHasChanged) { let options = { cssFiles: this.state.lastRenderCssFiles, + downloadResources: Setting.value('sync.resourceDownloadMode'), }; this.webviewRef_.current.wrappedInstance.send('setHtml', html, options); this.lastSetHtml_ = html; diff --git a/ElectronClient/app/gui/SideBar.jsx b/ElectronClient/app/gui/SideBar.jsx index 183bc98e0..1998c33d7 100644 --- a/ElectronClient/app/gui/SideBar.jsx +++ b/ElectronClient/app/gui/SideBar.jsx @@ -714,7 +714,7 @@ class SideBarComponent extends React.Component { let resourceFetcherText = ''; if (this.props.resourceFetcher && this.props.resourceFetcher.toFetchCount) { - resourceFetcherText = _('Fetching resources: %d', this.props.resourceFetcher.toFetchCount); + resourceFetcherText = _('Fetching resources: %d/%d', this.props.resourceFetcher.fetchingCount, this.props.resourceFetcher.toFetchCount); } let lines = Synchronizer.reportToLines(this.props.syncReport); diff --git a/ElectronClient/app/gui/note-viewer/index.html b/ElectronClient/app/gui/note-viewer/index.html index 93a00434c..b9d25593f 100644 --- a/ElectronClient/app/gui/note-viewer/index.html +++ b/ElectronClient/app/gui/note-viewer/index.html @@ -123,6 +123,10 @@ } loadCssFiles(event.options.cssFiles); + + if (event.options.downloadResources === 'manual') { + webviewLib.setupResourceManualDownload(); + } } let ignoreNextScrollEvent = false; @@ -308,7 +312,7 @@ updateBodyHeight(); }); - updateBodyHeight(); + updateBodyHeight(); diff --git a/ReactNativeClient/lib/BaseApplication.js b/ReactNativeClient/lib/BaseApplication.js index e1b962700..39bc751f7 100644 --- a/ReactNativeClient/lib/BaseApplication.js +++ b/ReactNativeClient/lib/BaseApplication.js @@ -279,6 +279,12 @@ class BaseApplication { } } + resourceFetcher_downloadComplete(event) { + if (event.encrypted) { + DecryptionWorker.instance().scheduleStart(); + } + } + reducerActionToString(action) { let o = [action.type]; if ('id' in action) o.push(action.id); @@ -426,7 +432,7 @@ class BaseApplication { } if (this.hasGui() && action.type === 'SYNC_CREATED_RESOURCE') { - ResourceFetcher.instance().queueDownload(action.id); + ResourceFetcher.instance().autoAddResources(); } if (refreshFolders) { @@ -603,6 +609,7 @@ class BaseApplication { ResourceFetcher.instance().setFileApi(() => { return reg.syncTarget().fileApi() }); ResourceFetcher.instance().setLogger(this.logger_); + ResourceFetcher.instance().on('downloadComplete', this.resourceFetcher_downloadComplete); ResourceFetcher.instance().start(); SearchEngine.instance().setDb(reg.db()); diff --git a/ReactNativeClient/lib/MdToHtml/noteStyle.js b/ReactNativeClient/lib/MdToHtml/noteStyle.js index 8657d2e18..fe49144e1 100644 --- a/ReactNativeClient/lib/MdToHtml/noteStyle.js +++ b/ReactNativeClient/lib/MdToHtml/noteStyle.js @@ -161,6 +161,21 @@ module.exports = function(style, options) { color: black; } + .not-loaded-resource img { + width: 1.15em; + height: 1.15em; + } + + a.not-loaded-resource img { + margin-right: .2em; + } + + a.not-loaded-resource { + display: flex; + flex-direction: row; + align-items: center; + } + .checkbox-label-checked { opacity: 0.5; } diff --git a/ReactNativeClient/lib/MdToHtml/rules/html_image.js b/ReactNativeClient/lib/MdToHtml/rules/html_image.js index dc4992477..6e3754f0d 100644 --- a/ReactNativeClient/lib/MdToHtml/rules/html_image.js +++ b/ReactNativeClient/lib/MdToHtml/rules/html_image.js @@ -5,8 +5,14 @@ const utils = require('../utils'); function renderImageHtml(before, src, after, ruleOptions) { const resourceId = Resource.urlToId(src); - const resource = ruleOptions.resources[resourceId]; - if (!resource) return '
' + utils.loaderImage() + '
'; + const result = ruleOptions.resources[resourceId]; + const resource = result ? result.item : null; + const resourceStatus = utils.resourceStatus(result); + + if (resourceStatus !== 'ready') { + const icon = utils.resourceStatusImage(resourceStatus); + return '
' + '' + '
'; + } const mime = resource.mime ? resource.mime.toLowerCase() : ''; if (Resource.isSupportedImageMimeType(mime)) { diff --git a/ReactNativeClient/lib/MdToHtml/rules/image.js b/ReactNativeClient/lib/MdToHtml/rules/image.js index e5905abf9..dfbbf69ba 100644 --- a/ReactNativeClient/lib/MdToHtml/rules/image.js +++ b/ReactNativeClient/lib/MdToHtml/rules/image.js @@ -14,8 +14,14 @@ function installRule(markdownIt, mdOptions, ruleOptions) { if (!Resource.isResourceUrl(src)) return defaultRender(tokens, idx, options, env, self); const resourceId = Resource.urlToId(src); - const resource = ruleOptions.resources[resourceId]; - if (!resource) return '
' + utils.loaderImage() + '
'; + const result = ruleOptions.resources[resourceId]; + const resource = result ? result.item : null; + const resourceStatus = utils.resourceStatus(result); + + if (resourceStatus !== 'ready') { + const icon = utils.resourceStatusImage(resourceStatus); + return '
' + '' + '
'; + } const mime = resource.mime ? resource.mime.toLowerCase() : ''; if (Resource.isSupportedImageMimeType(mime)) { diff --git a/ReactNativeClient/lib/MdToHtml/rules/link_open.js b/ReactNativeClient/lib/MdToHtml/rules/link_open.js index ef72dbf43..fb04f6d8d 100644 --- a/ReactNativeClient/lib/MdToHtml/rules/link_open.js +++ b/ReactNativeClient/lib/MdToHtml/rules/link_open.js @@ -18,9 +18,19 @@ function installRule(markdownIt, mdOptions, ruleOptions) { let hrefAttr = '#'; if (isResourceUrl) { const resourceId = Resource.pathToId(href); - href = "joplin://" + resourceId; - resourceIdAttr = "data-resource-id='" + resourceId + "'"; - icon = ''; + + const result = ruleOptions.resources[resourceId]; + const resource = result ? result.item : null; + const resourceStatus = utils.resourceStatus(result); + + if (resourceStatus !== 'ready') { + const icon = utils.resourceStatusFile(resourceStatus); + return '' + ''; + } else { + href = "joplin://" + resourceId; + resourceIdAttr = "data-resource-id='" + resourceId + "'"; + icon = ''; + } } else { // If the link is a plain URL (as opposed to a resource link), set the href to the actual // link. This allows the link to be exported too when exporting to PDF. @@ -29,7 +39,6 @@ function installRule(markdownIt, mdOptions, ruleOptions) { let js = ruleOptions.postMessageSyntax + "(" + JSON.stringify(href) + "); return false;"; if (hrefAttr.indexOf('#') === 0 && href.indexOf('#') === 0) js = ''; // If it's an internal anchor, don't add any JS since the webview is going to handle navigating to the right place - // if (js) hrefAttr = '#'; return "" + icon; }; } diff --git a/ReactNativeClient/lib/MdToHtml/utils.js b/ReactNativeClient/lib/MdToHtml/utils.js index 9b687e4e0..ccc391f78 100644 --- a/ReactNativeClient/lib/MdToHtml/utils.js +++ b/ReactNativeClient/lib/MdToHtml/utils.js @@ -1,14 +1,89 @@ -module.exports = { - - getAttr: function(attrs, name, defaultValue = null) { - for (let i = 0; i < attrs.length; i++) { - if (attrs[i][0] === name) return attrs[i].length > 1 ? attrs[i][1] : null; +const Resource = require('lib/models/Resource.js'); + +const utils = {}; + +utils.getAttr = function(attrs, name, defaultValue = null) { + for (let i = 0; i < attrs.length; i++) { + if (attrs[i][0] === name) return attrs[i].length > 1 ? attrs[i][1] : null; + } + return defaultValue; +} + +utils.notDownloadedResource = function() { + return ` + + + + `; +} + +utils.notDownloadedImage = function() { + // https://github.com/ForkAwesome/Fork-Awesome/blob/master/src/icons/svg/file-image-o.svg + // Height changed to 1795 + return ` + + + + `; +} + +utils.notDownloadedFile = function() { + // https://github.com/ForkAwesome/Fork-Awesome/blob/master/src/icons/svg/file-o.svg + return ` + + + + `; +} + +utils.errorImage = function() { + // https://github.com/ForkAwesome/Fork-Awesome/blob/master/src/icons/svg/times-circle.svg + return ` + + + + `; +} + +utils.loaderImage = function() { + return ''; +} + +utils.resourceStatusImage = function(state) { + if (state === 'notDownloaded') return utils.notDownloadedResource(); + return utils.resourceStatusFile(state); +} + +utils.resourceStatusFile = function(state) { + if (state === 'notDownloaded') return utils.notDownloadedResource(); + if (state === 'downloading') return utils.loaderImage(); + if (state === 'encrypted') return utils.loaderImage(); + if (state === 'error') return utils.errorImage(); + + throw new Error('Unknown state: ' + state); +} + +utils.resourceStatus = function(resourceInfo, localState) { + let resourceStatus = 'ready'; + + if (resourceInfo) { + const resource = resourceInfo.item; + const localState = resourceInfo.localState; + + if (localState.fetch_status === Resource.FETCH_STATUS_IDLE) { + resourceStatus = 'notDownloaded'; + } else if (localState.fetch_status === Resource.FETCH_STATUS_STARTED) { + resourceStatus = 'downloading'; + } else if (localState.fetch_status === Resource.FETCH_STATUS_DONE) { + if (resource.encryption_blob_encrypted || resource.encryption_applied) { + resourceStatus = 'encrypted'; + } } - return defaultValue; - }, + } else { + resourceStatus = 'notDownloaded'; + } - loaderImage: function() { - return ''; - }, + return resourceStatus; +} -}; \ No newline at end of file +module.exports = utils; \ No newline at end of file diff --git a/ReactNativeClient/lib/MdToHtml/webviewLib.js b/ReactNativeClient/lib/MdToHtml/webviewLib.js index 3a19d35c6..44792a687 100644 --- a/ReactNativeClient/lib/MdToHtml/webviewLib.js +++ b/ReactNativeClient/lib/MdToHtml/webviewLib.js @@ -1,7 +1,33 @@ const webviewLib = {}; +let manualDownloadResourceElements = []; + +webviewLib.onUnloadedResourceClick = function(event) { + const resourceId = event.currentTarget.getAttribute('data-resource-id'); + webviewLib.options_.postMessage('markForDownload:' + resourceId); +} + +webviewLib.setupResourceManualDownload = function() { + for (const element of manualDownloadResourceElements) { + element.style.cursor = 'default'; + element.removeEventListener('click', webviewLib.onUnloadedResourceClick); + } + + manualDownloadResourceElements = []; + + const elements = document.getElementsByClassName('resource-status-notDownloaded'); + + for (const element of elements) { + element.style.cursor = 'pointer'; + element.addEventListener('click', webviewLib.onUnloadedResourceClick); + manualDownloadResourceElements.push(element); + } +} + webviewLib.handleInternalLink = function(event, anchorNode) { const href = anchorNode.getAttribute('href'); + if (!href) return false; + if (href.indexOf('#') === 0) { event.preventDefault(); let old_hash = location.hash; @@ -18,6 +44,7 @@ webviewLib.handleInternalLink = function(event, anchorNode) { setTimeout(function() { location.hash = old_hash; }, 10); return true; } + return false; } @@ -52,9 +79,8 @@ document.addEventListener('click', function(event) { // as Katex. if (!anchor.hasAttribute('data-from-md')) { if (webviewLib.handleInternalLink(event, anchor)) return; - event.preventDefault(); - webviewLib.options_.postMessage(anchor.getAttribute('href')); + if (anchor.getAttribute('href')) webviewLib.options_.postMessage(anchor.getAttribute('href')); return; } @@ -63,3 +89,4 @@ document.addEventListener('click', function(event) { if (webviewLib.handleInternalLink(event, anchor)) return; } }); + diff --git a/ReactNativeClient/lib/components/note-body-viewer.js b/ReactNativeClient/lib/components/note-body-viewer.js index 7fb2cda10..8b90a30c7 100644 --- a/ReactNativeClient/lib/components/note-body-viewer.js +++ b/ReactNativeClient/lib/components/note-body-viewer.js @@ -104,11 +104,19 @@ class NoteBodyViewer extends Component { let result = this.mdToHtml_.render(bodyToRender, this.props.webViewStyle, mdOptions); let html = result.html; + const resourceDownloadMode = Setting.value('sync.resourceDownloadMode'); + const injectedJs = [this.mdToHtml_.injectedJavaScript()]; injectedJs.push(shim.injectedJs('webviewLib')); injectedJs.push('webviewLib.initialize({ postMessage: postMessage });'); - - console.info(injectedJs); + injectedJs.push(` + const readyStateCheckInterval = setInterval(function() { + if (document.readyState === "complete") { + clearInterval(readyStateCheckInterval); + if ("${resourceDownloadMode}" === "manual") webviewLib.setupResourceManualDownload(); + } + }, 10); + `); html = ` @@ -172,9 +180,10 @@ class NoteBodyViewer extends Component { if (msg.indexOf('checkboxclick:') === 0) { const newBody = shared.toggleCheckbox(msg, this.props.note.body); if (this.props.onCheckboxChange) this.props.onCheckboxChange(newBody); - } else if (msg.indexOf('bodyscroll:') === 0) { - //msg = msg.split(':'); - //this.bodyScrollTop_ = Number(msg[1]); + } else if (msg.indexOf('markForDownload:') === 0) { + msg = msg.split(':'); + const resourceId = msg[1]; + if (this.props.onMarkForDownload) this.props.onMarkForDownload({ resourceId: resourceId }); } else { this.props.onJoplinLinkClick(msg); } diff --git a/ReactNativeClient/lib/components/screens/note.js b/ReactNativeClient/lib/components/screens/note.js index 29890bbbd..0da2148f6 100644 --- a/ReactNativeClient/lib/components/screens/note.js +++ b/ReactNativeClient/lib/components/screens/note.js @@ -164,10 +164,11 @@ class NoteScreenComponent extends BaseScreenComponent { } } - this.resourceFetcher_downloadComplete = async (resource) => { + this.refreshResource = async (resource) => { if (!this.state.note || !this.state.note.body) return; const resourceIds = await Note.linkedResourceIds(this.state.note.body); if (resourceIds.indexOf(resource.id) >= 0 && this.refs.noteBodyViewer) { + shared.clearResourceCache(); const attachedResources = await shared.attachedResources(this.state.note.body); this.setState({ noteResources: attachedResources }, () => { this.refs.noteBodyViewer.rebuildMd(); @@ -178,6 +179,7 @@ class NoteScreenComponent extends BaseScreenComponent { this.takePhoto_onPress = this.takePhoto_onPress.bind(this); this.cameraView_onPhoto = this.cameraView_onPhoto.bind(this); this.cameraView_onCancel = this.cameraView_onCancel.bind(this); + this.onMarkForDownload = this.onMarkForDownload.bind(this); } styles() { @@ -235,13 +237,23 @@ class NoteScreenComponent extends BaseScreenComponent { BackButtonService.addHandler(this.backHandler); NavService.addHandler(this.navHandler); - ResourceFetcher.instance().on('downloadComplete', this.resourceFetcher_downloadComplete); + shared.clearResourceCache(); + shared.installResourceHandling(this.refreshResource); await shared.initState(this); + if (this.state.note && this.state.note.body && Setting.value('sync.resourceDownloadMode') === 'auto') { + const resourceIds = await Note.linkedResourceIds(this.state.note.body); + await ResourceFetcher.instance().markForDownload(resourceIds); + } + this.refreshNoteMetadata(); } + onMarkForDownload(event) { + ResourceFetcher.instance().markForDownload(event.resourceId); + } + refreshNoteMetadata(force = null) { return shared.refreshNoteMetadata(this, force); } @@ -250,7 +262,7 @@ class NoteScreenComponent extends BaseScreenComponent { BackButtonService.removeHandler(this.backHandler); NavService.removeHandler(this.navHandler); - ResourceFetcher.instance().off('downloadComplete', this.resourceFetcher_downloadComplete); + shared.uninstallResourceHandling(this.refreshResource); if (Platform.OS !== 'ios' && this.state.fromShare) { ShareExtension.close(); @@ -626,6 +638,7 @@ class NoteScreenComponent extends BaseScreenComponent { highlightedKeywords={keywords} theme={this.props.theme} onCheckboxChange={(newBody) => { onCheckboxChange(newBody) }} + onMarkForDownload={this.onMarkForDownload} onLoadEnd={() => { setTimeout(() => { this.setState({ HACK_webviewLoadingState: 1 }); diff --git a/ReactNativeClient/lib/components/shared/note-screen-shared.js b/ReactNativeClient/lib/components/shared/note-screen-shared.js index 0ab90f3be..3458026b8 100644 --- a/ReactNativeClient/lib/components/shared/note-screen-shared.js +++ b/ReactNativeClient/lib/components/shared/note-screen-shared.js @@ -3,6 +3,8 @@ const Folder = require('lib/models/Folder.js'); const BaseModel = require('lib/BaseModel.js'); const Note = require('lib/models/Note.js'); const Resource = require('lib/models/Resource.js'); +const ResourceFetcher = require('lib/services/ResourceFetcher.js'); +const DecryptionWorker = require('lib/services/DecryptionWorker.js'); const Setting = require('lib/models/Setting.js'); const Mutex = require('async-mutex').Mutex; @@ -154,7 +156,11 @@ shared.noteComponent_change = function(comp, propName, propValue) { comp.setState(newState); } -const resourceCache_ = {}; +let resourceCache_ = {}; + +shared.clearResourceCache = function() { + resourceCache_ = {}; +} shared.attachedResources = async function(noteBody) { if (!noteBody) return {}; @@ -163,14 +169,20 @@ shared.attachedResources = async function(noteBody) { const output = {}; for (let i = 0; i < resourceIds.length; i++) { const id = resourceIds[i]; + if (resourceCache_[id]) { output[id] = resourceCache_[id]; } else { const resource = await Resource.load(id); - const isReady = await Resource.isReady(resource); - if (!isReady) continue; - resourceCache_[id] = resource; - output[id] = resource; + const localState = await Resource.localState(resource); + + const o = { + item: resource, + localState: localState, + }; + + resourceCache_[id] = o; + output[id] = o; } } @@ -269,4 +281,16 @@ shared.toggleCheckbox = function(ipcMessage, noteBody) { return newBody.join('\n') } +shared.installResourceHandling = function(refreshResourceHandler) { + ResourceFetcher.instance().on('downloadComplete', refreshResourceHandler); + ResourceFetcher.instance().on('downloadStarted', refreshResourceHandler); + DecryptionWorker.instance().on('resourceDecrypted', refreshResourceHandler); +} + +shared.uninstallResourceHandling = function(refreshResourceHandler) { + ResourceFetcher.instance().off('downloadComplete', refreshResourceHandler); + ResourceFetcher.instance().off('downloadStarted', refreshResourceHandler); + DecryptionWorker.instance().off('resourceDecrypted', refreshResourceHandler); +} + module.exports = shared; diff --git a/ReactNativeClient/lib/components/shared/reduxSharedMiddleware.js b/ReactNativeClient/lib/components/shared/reduxSharedMiddleware.js index b4f0f4878..fae5fed54 100644 --- a/ReactNativeClient/lib/components/shared/reduxSharedMiddleware.js +++ b/ReactNativeClient/lib/components/shared/reduxSharedMiddleware.js @@ -1,6 +1,7 @@ const Setting = require('lib/models/Setting'); const Tag = require('lib/models/Tag'); const { reg } = require('lib/registry.js'); +const ResourceFetcher = require('lib/services/ResourceFetcher'); const reduxSharedMiddleware = async function(store, next, action) { const newState = store.getState(); @@ -15,6 +16,10 @@ const reduxSharedMiddleware = async function(store, next, action) { reg.resetSyncTarget(); } + if (action.type === 'SETTING_UPDATE_ONE' && action.key === 'sync.resourceDownloadMode') { + ResourceFetcher.instance().autoAddResources(); + } + if (action.type == 'NOTE_DELETE') { refreshTags = true; } diff --git a/ReactNativeClient/lib/components/side-menu-content.js b/ReactNativeClient/lib/components/side-menu-content.js index 9d81131e1..85b8a9692 100644 --- a/ReactNativeClient/lib/components/side-menu-content.js +++ b/ReactNativeClient/lib/components/side-menu-content.js @@ -207,13 +207,13 @@ class SideMenuContentComponent extends Component { let resourceFetcherText = ''; if (this.props.resourceFetcher && this.props.resourceFetcher.toFetchCount) { - resourceFetcherText = _('Fetching resources: %d', this.props.resourceFetcher.toFetchCount); + resourceFetcherText = _('Fetching resources: %d/%d', this.props.resourceFetcher.fetchingCount, this.props.resourceFetcher.toFetchCount); } let fullReport = []; if (syncReportText) fullReport.push(syncReportText); // if (fullReport.length) fullReport.push(''); - if (resourceFetcherText) lines.push(resourceFetcherText); + if (resourceFetcherText) fullReport.push(resourceFetcherText); if (decryptionReportText) fullReport.push(decryptionReportText); while (fullReport.length < 12) fullReport.push(''); // Add blank lines so that height of report text is fixed and doesn't affect scrolling diff --git a/ReactNativeClient/lib/joplin-database.js b/ReactNativeClient/lib/joplin-database.js index 9b60defd2..c66eaaa68 100644 --- a/ReactNativeClient/lib/joplin-database.js +++ b/ReactNativeClient/lib/joplin-database.js @@ -263,7 +263,7 @@ class JoplinDatabase extends Database { // must be set in the synchronizer too. // Note: v16 and v17 don't do anything. They were used to debug an issue. - const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]; + const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]; let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion); @@ -564,6 +564,21 @@ class JoplinDatabase extends Database { queries.push('ALTER TABLE sync_items ADD COLUMN item_location INT NOT NULL DEFAULT 1'); } + if (targetVersion == 22) { + const newTableSql = ` + CREATE TABLE resources_to_download ( + id INTEGER PRIMARY KEY, + resource_id TEXT NOT NULL, + updated_time INT NOT NULL, + created_time INT NOT NULL + ); + `; + queries.push(this.sqlStringToLines(newTableSql)[0]); + + queries.push('CREATE INDEX resources_to_download_resource_id ON resources_to_download (resource_id)'); + queries.push('CREATE INDEX resources_to_download_updated_time ON resources_to_download (updated_time)'); + } + queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] }); try { diff --git a/ReactNativeClient/lib/models/BaseItem.js b/ReactNativeClient/lib/models/BaseItem.js index db0517196..3618e8254 100644 --- a/ReactNativeClient/lib/models/BaseItem.js +++ b/ReactNativeClient/lib/models/BaseItem.js @@ -458,7 +458,13 @@ class BaseItem extends BaseModel { const className = classNames[i]; const ItemClass = this.getClass(className); - const whereSql = className === 'Resource' ? ['(encryption_blob_encrypted = 1 OR encryption_applied = 1)'] : ['encryption_applied = 1']; + let whereSql = ['encryption_applied = 1']; + + if (className === 'Resource') { + const blobDownloadedButEncryptedSql = 'encryption_blob_encrypted = 1 AND id IN (SELECT resource_id FROM resource_local_states WHERE fetch_status = 2))'; + whereSql = ['(encryption_applied = 1 OR (' + blobDownloadedButEncryptedSql + ')']; + } + if (exclusions.length) whereSql.push('id NOT IN ("' + exclusions.join('","') + '")'); const sql = sprintf(` diff --git a/ReactNativeClient/lib/models/Resource.js b/ReactNativeClient/lib/models/Resource.js index 7427ec976..fbd75b332 100644 --- a/ReactNativeClient/lib/models/Resource.js +++ b/ReactNativeClient/lib/models/Resource.js @@ -32,15 +32,14 @@ class Resource extends BaseItem { return imageMimeTypes.indexOf(type.toLowerCase()) >= 0; } - static needToBeFetched(limit = null) { - let sql = 'SELECT * FROM resources WHERE id IN (SELECT resource_id FROM resource_local_states WHERE fetch_status = ?) ORDER BY updated_time DESC'; - if (limit !== null) sql += ' LIMIT ' + limit; - return this.modelSelectAll(sql, [Resource.FETCH_STATUS_IDLE]); - } - - static async needToBeFetchedCount() { - const r = await this.db().selectOne('SELECT count(*) as total FROM resource_local_states WHERE fetch_status = ?', [Resource.FETCH_STATUS_IDLE]); - return r ? r['total'] : 0; + static needToBeFetched(resourceDownloadMode = null, limit = null) { + let sql = ['SELECT * FROM resources WHERE encryption_applied = 0 AND id IN (SELECT resource_id FROM resource_local_states WHERE fetch_status = ?)']; + if (resourceDownloadMode !== 'always') { + sql.push('AND resources.id IN (SELECT resource_id FROM resources_to_download)'); + } + sql.push('ORDER BY updated_time DESC'); + if (limit !== null) sql.push('LIMIT ' + limit); + return this.modelSelectAll(sql.join(' '), [Resource.FETCH_STATUS_IDLE]); } static async resetStartedFetchStatus() { @@ -52,13 +51,6 @@ class Resource extends BaseItem { return Resource.fsDriver_; } - static filename(resource, encryptedBlob = false) { - let extension = encryptedBlob ? 'crypted' : resource.file_extension; - if (!extension) extension = resource.mime ? mime.toFileExtension(resource.mime) : ''; - extension = extension ? ('.' + extension) : ''; - return resource.id + extension; - } - static friendlyFilename(resource) { let output = safeFilename(resource.title); // Make sure not to allow spaces or any special characters as it's not supported in HTTP headers if (!output) output = resource.id; @@ -76,6 +68,13 @@ class Resource extends BaseItem { return Setting.value('resourceDirName'); } + static filename(resource, encryptedBlob = false) { + let extension = encryptedBlob ? 'crypted' : resource.file_extension; + if (!extension) extension = resource.mime ? mime.toFileExtension(resource.mime) : ''; + extension = extension ? ('.' + extension) : ''; + return resource.id + extension; + } + static relativePath(resource, encryptedBlob = false) { return Setting.value('resourceDirName') + '/' + this.filename(resource, encryptedBlob); } @@ -96,6 +95,13 @@ class Resource extends BaseItem { const decryptedItem = item.encryption_cipher_text ? await super.decrypt(item) : Object.assign({}, item); if (!decryptedItem.encryption_blob_encrypted) return decryptedItem; + const localState = await this.localState(item); + if (localState.fetch_status !== Resource.FETCH_STATUS_DONE) { + // Not an error - it means the blob has not been downloaded yet. + // It will be decrypted later on, once downloaded. + return decryptedItem; + } + const plainTextPath = this.fullPath(decryptedItem); const encryptedPath = this.fullPath(decryptedItem, true); const noExtPath = pathUtils.dirname(encryptedPath) + '/' + pathUtils.filename(encryptedPath); @@ -109,12 +115,7 @@ class Resource extends BaseItem { } try { - // const stat = await this.fsDriver().stat(encryptedPath); - await this.encryptionService().decryptFile(encryptedPath, plainTextPath, { - // onProgress: (progress) => { - // console.info('Decryption: ', progress.doneSize / stat.size); - // }, - }); + await this.encryptionService().decryptFile(encryptedPath, plainTextPath); } catch (error) { if (error.code === 'invalidIdentifier') { // As the identifier is invalid it most likely means that this is not encrypted data @@ -149,12 +150,7 @@ class Resource extends BaseItem { if (resource.encryption_blob_encrypted) return { path: encryptedPath, resource: resource }; try { - // const stat = await this.fsDriver().stat(plainTextPath); - await this.encryptionService().encryptFile(plainTextPath, encryptedPath, { - // onProgress: (progress) => { - // console.info(progress.doneSize / stat.size); - // }, - }); + await this.encryptionService().encryptFile(plainTextPath, encryptedPath); } catch (error) { if (error.code === 'ENOENT') throw new JoplinError('File not found:' + error.toString(), 'fileNotFound'); throw error; @@ -244,6 +240,20 @@ class Resource extends BaseItem { await ResourceLocalState.batchDelete(ids); } + static async markForDownload(resourceId) { + // Insert the row only if it's not already there + const t = Date.now(); + await this.db().exec('INSERT INTO resources_to_download (resource_id, updated_time, created_time) SELECT ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM resources_to_download WHERE resource_id = ?)', [resourceId, t, t, resourceId]); + } + + static async downloadedButEncryptedBlobCount() { + const r = await this.db().selectOne('SELECT count(*) as total FROM resource_local_states WHERE fetch_status = ? AND resource_id IN (SELECT id FROM resources WHERE encryption_blob_encrypted = 1)', [ + Resource.FETCH_STATUS_DONE, + ]); + + return r ? r.total : 0; + } + } Resource.IMAGE_MAX_DIMENSION = 1920; diff --git a/ReactNativeClient/lib/models/Setting.js b/ReactNativeClient/lib/models/Setting.js index 0481dd811..555f0065c 100644 --- a/ReactNativeClient/lib/models/Setting.js +++ b/ReactNativeClient/lib/models/Setting.js @@ -155,6 +155,14 @@ class Setting extends BaseModel { return SyncTargetRegistry.idAndLabelPlainObject(); }}, + 'sync.resourceDownloadMode': { value: 'always', type: Setting.TYPE_STRING, section: 'sync', public: true, isEnum: true, appTypes: ['mobile', 'desktop'], label: () => _('Attachment download behaviour'), description: () => _('In "Manual" mode, attachments are downloaded only when you click on them. In "Auto", they are downloaded when you open the note. In "Always", all the attachments are downloaded whether you open the note or not.'), options: () => { + return { + 'always': _('Always'), + 'manual': _('Manual'), + 'auto': _('Auto'), + }; + }}, + 'sync.2.path': { value: '', type: Setting.TYPE_STRING, section:'sync', show: (settings) => { try { return settings['sync.target'] == SyncTargetRegistry.nameToId('filesystem') diff --git a/ReactNativeClient/lib/services/DecryptionWorker.js b/ReactNativeClient/lib/services/DecryptionWorker.js index b5ec01cbe..d8758db38 100644 --- a/ReactNativeClient/lib/services/DecryptionWorker.js +++ b/ReactNativeClient/lib/services/DecryptionWorker.js @@ -2,6 +2,7 @@ const BaseItem = require('lib/models/BaseItem'); const Resource = require('lib/models/Resource'); const ResourceService = require('lib/services/ResourceService'); const { Logger } = require('lib/logger.js'); +const EventEmitter = require('events'); class DecryptionWorker { @@ -14,6 +15,7 @@ class DecryptionWorker { }; this.scheduleId_ = null; + this.eventEmitter_ = new EventEmitter(); } setLogger(l) { @@ -24,6 +26,14 @@ class DecryptionWorker { return this.logger_; } + on(eventName, callback) { + return this.eventEmitter_.on(eventName, callback); + } + + off(eventName, callback) { + return this.eventEmitter_.removeListener(eventName, callback); + } + static instance() { if (this.instance_) return this.instance_; this.instance_ = new DecryptionWorker(); @@ -85,14 +95,6 @@ class DecryptionWorker { const ItemClass = BaseItem.itemClass(item); - if (item.type_ === Resource.modelType()) { - const ls = await Resource.localState(item); - if (ls.fetch_status !== Resource.FETCH_STATUS_DONE) { - excludedIds.push(item.id); - continue; - } - } - this.dispatchReport({ itemIndex: i, itemCount: items.length, @@ -101,10 +103,21 @@ class DecryptionWorker { // Don't log in production as it results in many messages when importing many items // this.logger().info('DecryptionWorker: decrypting: ' + item.id + ' (' + ItemClass.tableName() + ')'); try { - await ItemClass.decrypt(item); + const decryptedItem = await ItemClass.decrypt(item); + + if (decryptedItem.type_ === Resource.modelType() && !!decryptedItem.encryption_blob_encrypted) { + // itemsThatNeedDecryption() will return the resource again if the blob has not been decrypted, + // but that will result in an infinite loop if the blob simply has not been downloaded yet. + // So skip the ID for now, and the service will try to decrypt the blob again the next time. + excludedIds.push(decryptedItem.id); + } + + if (decryptedItem.type_ === Resource.modelType() && !decryptedItem.encryption_blob_encrypted) { + this.eventEmitter_.emit('resourceDecrypted', { id: decryptedItem.id }); + } } catch (error) { excludedIds.push(item.id); - + if (error.code === 'masterKeyNotLoaded' && options.masterKeyNotLoadedHandler === 'dispatch') { if (notLoadedMasterKeyDisptaches.indexOf(error.masterKeyId) < 0) { this.dispatch({ @@ -139,9 +152,16 @@ class DecryptionWorker { this.logger().info('DecryptionWorker: completed decryption.'); + const downloadedButEncryptedBlobCount = await Resource.downloadedButEncryptedBlobCount(); + this.dispatchReport({ state: 'idle' }); this.state_ = 'idle'; + + if (downloadedButEncryptedBlobCount) { + this.logger().info('DecryptionWorker: Some resources have been downloaded but are not decrypted yet. Scheduling another decryption. Resource count: ' + downloadedButEncryptedBlobCount); + this.scheduleStart(); + } } } diff --git a/ReactNativeClient/lib/services/ResourceFetcher.js b/ReactNativeClient/lib/services/ResourceFetcher.js index 021cb71b8..bdea59c74 100644 --- a/ReactNativeClient/lib/services/ResourceFetcher.js +++ b/ReactNativeClient/lib/services/ResourceFetcher.js @@ -1,4 +1,5 @@ const Resource = require('lib/models/Resource'); +const Setting = require('lib/models/Setting'); const BaseService = require('lib/services/BaseService'); const ResourceService = require('lib/services/ResourceService'); const BaseSyncTarget = require('lib/BaseSyncTarget'); @@ -63,23 +64,32 @@ class ResourceFetcher extends BaseService { } updateReport() { - if (this.updateReportIID_) return; - - this.updateReportIID_ = setTimeout(async () => { - const toFetchCount = await Resource.needToBeFetchedCount(); - this.dispatch({ - type: 'RESOURCE_FETCHER_SET', - toFetchCount: toFetchCount, - }); - this.updateReportIID_ = null; - }, 2000); + const fetchingCount = Object.keys(this.fetchingItems_).length; + this.dispatch({ + type: 'RESOURCE_FETCHER_SET', + fetchingCount: fetchingCount, + toFetchCount: fetchingCount + this.queue_.length, + }); } - queueDownload(resourceId, priority = null) { + async markForDownload(resourceIds) { + if (!Array.isArray(resourceIds)) resourceIds = [resourceIds]; + + for (const id of resourceIds) { + await Resource.markForDownload(id); + } + + for (const id of resourceIds) { + this.queueDownload_(id, 'high'); + } + } + + queueDownload_(resourceId, priority = null) { if (priority === null) priority = 'normal'; const index = this.queuedItemIndex_(resourceId); if (index >= 0) return false; + if (this.fetchingItems_[resourceId]) return false; const item = { id: resourceId }; @@ -99,6 +109,8 @@ class ResourceFetcher extends BaseService { if (this.fetchingItems_[resourceId]) return; this.fetchingItems_[resourceId] = true; + this.updateReport(); + const resource = await Resource.load(resourceId); const localState = await Resource.localState(resource); @@ -118,7 +130,7 @@ class ResourceFetcher extends BaseService { // might still be encrypted and the caller usually can't do much with this. In particular // the note being displayed will refresh the resource images but since they are still // encrypted it's not useful. Probably, the views should listen to DecryptionWorker events instead. - if (emitDownloadComplete) this.eventEmitter_.emit('downloadComplete', { id: resource.id }); + if (resource && emitDownloadComplete) this.eventEmitter_.emit('downloadComplete', { id: resource.id, encrypted: !!resource.encryption_blob_encrypted }); this.updateReport(); } @@ -137,7 +149,7 @@ class ResourceFetcher extends BaseService { this.fetchingItems_[resourceId] = resource; - const localResourceContentPath = Resource.fullPath(resource); + const localResourceContentPath = Resource.fullPath(resource, !!resource.encryption_blob_encrypted); const remoteResourceContentPath = this.resourceDirName_ + "/" + resource.id; await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_STARTED }); @@ -146,6 +158,8 @@ class ResourceFetcher extends BaseService { this.logger().debug('ResourceFetcher: Downloading resource: ' + resource.id); + this.eventEmitter_.emit('downloadStarted', { id: resource.id }) + fileApi.get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" }).then(async () => { await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_DONE }); this.logger().debug('ResourceFetcher: Resource downloaded: ' + resource.id); @@ -184,10 +198,12 @@ class ResourceFetcher extends BaseService { if (this.addingResources_) return; this.addingResources_ = true; + this.logger().info('ResourceFetcher: Auto-add resources: Mode: ' + Setting.value('sync.resourceDownloadMode')); + let count = 0; - const resources = await Resource.needToBeFetched(limit); + const resources = await Resource.needToBeFetched(Setting.value('sync.resourceDownloadMode'), limit); for (let i = 0; i < resources.length; i++) { - const added = this.queueDownload(resources[i].id); + const added = this.queueDownload_(resources[i].id); if (added) count++; } diff --git a/ReactNativeClient/root.js b/ReactNativeClient/root.js index ed876414e..c02e35abc 100644 --- a/ReactNativeClient/root.js +++ b/ReactNativeClient/root.js @@ -146,7 +146,7 @@ const generalMiddleware = store => next => async (action) => { } if (action.type === 'SYNC_CREATED_RESOURCE') { - ResourceFetcher.instance().queueDownload(action.id); + ResourceFetcher.instance().autoAddResources(); } return result; @@ -333,6 +333,12 @@ const appReducer = (state = appDefaultState, action) => { let store = createStore(appReducer, applyMiddleware(generalMiddleware)); storeDispatch = store.dispatch; +function resourceFetcher_downloadComplete(event) { + if (event.encrypted) { + DecryptionWorker.instance().scheduleStart(); + } +} + async function initialize(dispatch) { shimInit(); @@ -402,7 +408,34 @@ async function initialize(dispatch) { if (Setting.value('env') == 'prod') { await db.open({ name: 'joplin.sqlite' }) } else { - await db.open({ name: 'joplin-68.sqlite' }) + await db.open({ name: 'joplin-68.sqlite' }); + + const tableNames = [ + 'notes', + 'folders', + 'resources', + 'tags', + 'note_tags', + // 'master_keys', + 'item_changes', + 'note_resources', + // 'settings', + 'deleted_items', + 'sync_items', + 'notes_normalized', + 'revisions', + 'resources_to_download', + ]; + + const queries = []; + for (const n of tableNames) { + queries.push('DELETE FROM ' + n); + queries.push('DELETE FROM sqlite_sequence WHERE name="' + n + '"'); // Reset autoincremented IDs + } + + queries.push('DELETE FROM settings WHERE key="sync.7.context"'); + + // await db.transactionExecBatch(queries); } reg.logger().info('Database is ready.'); @@ -423,6 +456,10 @@ async function initialize(dispatch) { reg.logger().info('db.ftsEnabled = ', Setting.value('db.ftsEnabled')); } + if (Setting.value('env') === 'dev') { + Setting.setValue('welcome.enabled', false); + } + BaseItem.revisionService_ = RevisionService.instance(); // Note: for now we hard-code the folder sort order as we need to @@ -505,6 +542,8 @@ async function initialize(dispatch) { ResourceFetcher.instance().setFileApi(() => { return reg.syncTarget().fileApi() }); ResourceFetcher.instance().setLogger(reg.logger()); + ResourceFetcher.instance().dispatch = dispatch; + ResourceFetcher.instance().on('downloadComplete', resourceFetcher_downloadComplete); ResourceFetcher.instance().start(); SearchEngine.instance().setDb(reg.db());