diff --git a/CliClient/package-lock.json b/CliClient/package-lock.json index b4c7f032d..c9e3113ff 100644 --- a/CliClient/package-lock.json +++ b/CliClient/package-lock.json @@ -744,6 +744,11 @@ "resolved": "https://registry.npmjs.org/file-type/-/file-type-4.4.0.tgz", "integrity": "sha1-G2AOX8ofvcboDApwxxyNul95BsU=" }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "find-up": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", diff --git a/CliClient/package.json b/CliClient/package.json index 619dbdc90..92e47a3b9 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -35,6 +35,7 @@ "diacritics": "^1.3.0", "diff-match-patch": "^1.0.4", "es6-promise-pool": "^2.5.0", + "file-uri-to-path": "^1.0.0", "follow-redirects": "^1.2.4", "form-data": "^2.1.4", "fs-extra": "^5.0.0", diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index 4ee20b58f..9cbd1b129 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -899,6 +899,29 @@ describe('Synchronizer', function() { expect(ls.fetch_error).toBe('did not work'); })); + it('should set the resource file size if it is missing', asyncTest(async () => { + while (insideBeforeEach) await time.msleep(500); + + let folder1 = await Folder.save({ title: "folder1" }); + let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg'); + await synchronizer().start(); + + await switchClient(2); + + await synchronizer().start(); + let r1 = (await Resource.all())[0]; + await Resource.setFileSizeOnly(r1.id, -1); + r1 = await Resource.load(r1.id); + expect(r1.size).toBe(-1); + + const fetcher = new ResourceFetcher(() => { return synchronizer().api() }); + fetcher.queueDownload(r1.id); + await fetcher.waitForAllFinished(); + r1 = await Resource.load(r1.id); + expect(r1.size).toBe(2720); + })); + it('should delete resources', asyncTest(async () => { while (insideBeforeEach) await time.msleep(500); diff --git a/CliClient/tests/test-utils.js b/CliClient/tests/test-utils.js index 74c882b04..d3eed9b43 100644 --- a/CliClient/tests/test-utils.js +++ b/CliClient/tests/test-utils.js @@ -119,6 +119,7 @@ async function switchClient(id) { Note.db_ = databases_[id]; BaseItem.db_ = databases_[id]; Setting.db_ = databases_[id]; + Resource.db_ = databases_[id]; BaseItem.encryptionService_ = encryptionServices_[id]; Resource.encryptionService_ = encryptionServices_[id]; diff --git a/ElectronClient/app/app.js b/ElectronClient/app/app.js index 301d83d03..4bf74f07c 100644 --- a/ElectronClient/app/app.js +++ b/ElectronClient/app/app.js @@ -30,6 +30,7 @@ const Menu = bridge().Menu; const MenuItem = bridge().MenuItem; const PluginManager = require('lib/services/PluginManager'); const RevisionService = require('lib/services/RevisionService'); +const MigrationService = require('lib/services/MigrationService'); const pluginClasses = [ require('./plugins/GotoAnything.min'), @@ -1037,6 +1038,7 @@ class Application extends BaseApplication { // Make it available to the console window - useful to call revisionService.collectRevisions() window.revisionService = RevisionService.instance(); + window.migrationService = MigrationService.instance(); } } diff --git a/ReactNativeClient/android/app/build.gradle b/ReactNativeClient/android/app/build.gradle index dd0bc7d4d..50a5705c8 100644 --- a/ReactNativeClient/android/app/build.gradle +++ b/ReactNativeClient/android/app/build.gradle @@ -90,8 +90,8 @@ android { applicationId "net.cozic.joplin" minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion - versionCode 2097482 - versionName "1.0.246" + versionCode 2097483 + versionName "1.0.247" ndk { abiFilters "armeabi-v7a", "x86" } diff --git a/ReactNativeClient/lib/migrations/20.js b/ReactNativeClient/lib/migrations/20.js index 7e5b5b63f..674c0dfb8 100644 --- a/ReactNativeClient/lib/migrations/20.js +++ b/ReactNativeClient/lib/migrations/20.js @@ -2,6 +2,7 @@ const Resource = require('lib/models/Resource'); const Setting = require('lib/models/Setting'); const { shim } = require('lib/shim'); const { reg } = require('lib/registry.js'); +const { fileExtension } = require('lib/path-utils.js'); const script = {}; @@ -10,7 +11,10 @@ script.exec = async function() { const queries = []; for (const stat of stats) { + if (fileExtension(stat.path) === 'crypted') continue; const resourceId = Resource.pathToId(stat.path); + if (!resourceId) continue; + queries.push({ sql: 'UPDATE resources SET `size` = ? WHERE id = ?', params: [stat.size, resourceId] }); if (queries.length >= 1000) { diff --git a/ReactNativeClient/lib/models/Resource.js b/ReactNativeClient/lib/models/Resource.js index a12ba1260..2219fbcdc 100644 --- a/ReactNativeClient/lib/models/Resource.js +++ b/ReactNativeClient/lib/models/Resource.js @@ -214,6 +214,13 @@ class Resource extends BaseItem { await ResourceLocalState.save(Object.assign({}, state, { resource_id: id })); } + // Only set the `size` field and nothing else, not even the update_time + // This is because it's only necessary to do it once after migration 20 + // and each client does it so there's no need to sync the resource. + static async setFileSizeOnly(resourceId, fileSize) { + return this.db().exec('UPDATE resources set `size` = ? WHERE id = ?', [fileSize, resourceId]); + } + static async batchDelete(ids, options = null) { // For resources, there's not really batch deleting since there's the file data to delete // too, so each is processed one by one with the item being deleted last (since the db diff --git a/ReactNativeClient/lib/services/MigrationService.js b/ReactNativeClient/lib/services/MigrationService.js index e83b5bc0f..0d0fc91a4 100644 --- a/ReactNativeClient/lib/services/MigrationService.js +++ b/ReactNativeClient/lib/services/MigrationService.js @@ -14,16 +14,19 @@ class MigrationService extends BaseService { return this.instance_; } + async runScript(num) { + const script = Migration.script(num); + await script.exec(); + } + async run() { const migrations = await Migration.migrationsToDo(); for (const migration of migrations) { this.logger().info('Running migration: ' + migration.number); - const script = Migration.script(migration.number); - try { - await script.exec(); + await this.runScript(migration.number); await Migration.delete(migration.id); } catch (error) { this.logger().error('Cannot run migration: ' + migration.number, error); diff --git a/ReactNativeClient/lib/services/ResourceFetcher.js b/ReactNativeClient/lib/services/ResourceFetcher.js index b1278eda4..dbcfc9a04 100644 --- a/ReactNativeClient/lib/services/ResourceFetcher.js +++ b/ReactNativeClient/lib/services/ResourceFetcher.js @@ -3,6 +3,7 @@ const BaseService = require('lib/services/BaseService'); const BaseSyncTarget = require('lib/BaseSyncTarget'); const { Logger } = require('lib/logger.js'); const EventEmitter = require('events'); +const { shim } = require('lib/shim'); class ResourceFetcher extends BaseService { @@ -97,7 +98,17 @@ class ResourceFetcher extends BaseService { if (this.fetchingItems_[resourceId]) return; this.fetchingItems_[resourceId] = true; - const completeDownload = (emitDownloadComplete = true) => { + const completeDownload = async (emitDownloadComplete = true, localResourceContentPath = '') => { + + // 2019-05-12: This is only necessary to set the file size of the resources that come via + // sync. The other ones have been done using migrations/20.js. This code can be removed + // after a few months. + if (resource.size < 0 && localResourceContentPath) { + const itDoes = await shim.fsDriver().waitTillExists(localResourceContentPath); + const fileStat = await shim.fsDriver().stat(localResourceContentPath); + await Resource.setFileSizeOnly(resource.id, fileStat.size); + } + delete this.fetchingItems_[resource.id]; this.scheduleQueueProcess(); if (emitDownloadComplete) this.eventEmitter_.emit('downloadComplete', { id: resource.id }); @@ -110,7 +121,7 @@ class ResourceFetcher extends BaseService { // Shouldn't happen, but just to be safe don't re-download the // resource if it's already been downloaded. if (localState.fetch_status === Resource.FETCH_STATUS_DONE) { - completeDownload(false); + await completeDownload(false); return; } @@ -128,11 +139,11 @@ class ResourceFetcher extends BaseService { fileApi.get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" }).then(async () => { await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_DONE }); this.logger().debug('ResourceFetcher: Resource downloaded: ' + resource.id); - completeDownload(); + await completeDownload(true, localResourceContentPath); }).catch(async (error) => { this.logger().error('ResourceFetcher: Could not download resource: ' + resource.id, error); await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_ERROR, fetch_error: error.message }); - completeDownload(); + await completeDownload(); }); }