All: Make sure resource filesize is set in all cases

CliClient/package-lock.json

@@ -744,6 +744,11 @@
       "resolved": "https://registry.npmjs.org/file-type/-/file-type-4.4.0.tgz",
       "integrity": "sha1-G2AOX8ofvcboDApwxxyNul95BsU="
     },
+    "file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
+    },
     "find-up": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",

CliClient/package.json

@@ -35,6 +35,7 @@
     "diacritics": "^1.3.0",
     "diff-match-patch": "^1.0.4",
     "es6-promise-pool": "^2.5.0",
+    "file-uri-to-path": "^1.0.0",
     "follow-redirects": "^1.2.4",
     "form-data": "^2.1.4",
     "fs-extra": "^5.0.0",

CliClient/tests/synchronizer.js

@@ -899,6 +899,29 @@ describe('Synchronizer', function() {
 		expect(ls.fetch_error).toBe('did not work');
 	}));
 
+	it('should set the resource file size if it is missing', asyncTest(async () => {
+		while (insideBeforeEach) await time.msleep(500);
+
+		let folder1 = await Folder.save({ title: "folder1" });
+		let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+		await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg');
+		await synchronizer().start();	
+
+		await switchClient(2);
+
+		await synchronizer().start();
+		let r1 = (await Resource.all())[0];
+		await Resource.setFileSizeOnly(r1.id, -1);
+		r1 = await Resource.load(r1.id);
+		expect(r1.size).toBe(-1);
+
+		const fetcher = new ResourceFetcher(() => { return synchronizer().api() });
+		fetcher.queueDownload(r1.id);
+		await fetcher.waitForAllFinished();
+		r1 = await Resource.load(r1.id);
+		expect(r1.size).toBe(2720);
+	}));
+
 	it('should delete resources', asyncTest(async () => {
 		while (insideBeforeEach) await time.msleep(500);
 

CliClient/tests/test-utils.js

@@ -119,6 +119,7 @@ async function switchClient(id) {
 	Note.db_ = databases_[id];
 	BaseItem.db_ = databases_[id];
 	Setting.db_ = databases_[id];
+	Resource.db_ = databases_[id];
 
 	BaseItem.encryptionService_ = encryptionServices_[id];
 	Resource.encryptionService_ = encryptionServices_[id];

ElectronClient/app/app.js

@@ -30,6 +30,7 @@ const Menu = bridge().Menu;
 const MenuItem = bridge().MenuItem;
 const PluginManager = require('lib/services/PluginManager');
 const RevisionService = require('lib/services/RevisionService');
+const MigrationService = require('lib/services/MigrationService');
 
 const pluginClasses = [
 	require('./plugins/GotoAnything.min'),
@@ -1037,6 +1038,7 @@ class Application extends BaseApplication {
 
 		// Make it available to the console window - useful to call revisionService.collectRevisions()
 		window.revisionService = RevisionService.instance();
+		window.migrationService = MigrationService.instance();
 	}
 
 }

ReactNativeClient/android/app/build.gradle

@@ -90,8 +90,8 @@ android {
 		applicationId "net.cozic.joplin"
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
-		versionCode 2097482
-		versionName "1.0.246"
+		versionCode 2097483
+		versionName "1.0.247"
 		ndk {
 			abiFilters "armeabi-v7a", "x86"
 		}

ReactNativeClient/lib/migrations/20.js

@@ -2,6 +2,7 @@ const Resource = require('lib/models/Resource');
 const Setting = require('lib/models/Setting');
 const { shim } = require('lib/shim');
 const { reg } = require('lib/registry.js');
+const { fileExtension } = require('lib/path-utils.js');
 
 const script = {};
 
@@ -10,7 +11,10 @@ script.exec = async function() {
 
 	const queries = [];
 	for (const stat of stats) {
+		if (fileExtension(stat.path) === 'crypted') continue;		
 		const resourceId = Resource.pathToId(stat.path);
+		if (!resourceId) continue;
+
 		queries.push({ sql: 'UPDATE resources SET `size` = ? WHERE id = ?', params: [stat.size, resourceId] });
 
 		if (queries.length >= 1000) {

ReactNativeClient/lib/models/Resource.js

@@ -214,6 +214,13 @@ class Resource extends BaseItem {
 		await ResourceLocalState.save(Object.assign({}, state, { resource_id: id }));
 	}
 
+	// Only set the `size` field and nothing else, not even the update_time
+	// This is because it's only necessary to do it once after migration 20
+	// and each client does it so there's no need to sync the resource.
+	static async setFileSizeOnly(resourceId, fileSize) {
+		return this.db().exec('UPDATE resources set `size` = ? WHERE id = ?', [fileSize, resourceId]);
+	}
+
 	static async batchDelete(ids, options = null) {
 		// For resources, there's not really batch deleting since there's the file data to delete
 		// too, so each is processed one by one with the item being deleted last (since the db

ReactNativeClient/lib/services/MigrationService.js

@@ -14,16 +14,19 @@ class MigrationService extends BaseService {
 		return this.instance_;
 	}
 
+	async runScript(num) {
+		const script = Migration.script(num);
+		await script.exec();
+	}
+
 	async run() {
 		const migrations = await Migration.migrationsToDo();
 
 		for (const migration of migrations) {
 			this.logger().info('Running migration: ' + migration.number);
 
-			const script = Migration.script(migration.number);
-
 			try {
-				await script.exec();
+				await this.runScript(migration.number);
 				await Migration.delete(migration.id);
 			} catch (error) {
 				this.logger().error('Cannot run migration: ' + migration.number, error);

ReactNativeClient/lib/services/ResourceFetcher.js

@@ -3,6 +3,7 @@ const BaseService = require('lib/services/BaseService');
 const BaseSyncTarget = require('lib/BaseSyncTarget');
 const { Logger } = require('lib/logger.js');
 const EventEmitter = require('events');
+const { shim } = require('lib/shim');
 
 class ResourceFetcher extends BaseService {
 
@@ -97,7 +98,17 @@ class ResourceFetcher extends BaseService {
 		if (this.fetchingItems_[resourceId]) return;
 		this.fetchingItems_[resourceId] = true;
 
-		const completeDownload = (emitDownloadComplete = true) => {
+		const completeDownload = async (emitDownloadComplete = true, localResourceContentPath = '') => {
+
+			// 2019-05-12: This is only necessary to set the file size of the resources that come via
+			// sync. The other ones have been done using migrations/20.js. This code can be removed
+			// after a few months.
+			if (resource.size < 0 && localResourceContentPath) {
+				const itDoes = await shim.fsDriver().waitTillExists(localResourceContentPath);
+				const fileStat = await shim.fsDriver().stat(localResourceContentPath);
+				await Resource.setFileSizeOnly(resource.id, fileStat.size);
+			}
+
 			delete this.fetchingItems_[resource.id];
 			this.scheduleQueueProcess();
 			if (emitDownloadComplete) this.eventEmitter_.emit('downloadComplete', { id: resource.id });
@@ -110,7 +121,7 @@ class ResourceFetcher extends BaseService {
 		// Shouldn't happen, but just to be safe don't re-download the
 		// resource if it's already been downloaded.
 		if (localState.fetch_status === Resource.FETCH_STATUS_DONE) {
-			completeDownload(false);
+			await completeDownload(false);
 			return;
 		}
 
@@ -128,11 +139,11 @@ class ResourceFetcher extends BaseService {
 		fileApi.get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" }).then(async () => {
 			await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_DONE });
 			this.logger().debug('ResourceFetcher: Resource downloaded: ' + resource.id);
-			completeDownload();
+			await completeDownload(true, localResourceContentPath);
 		}).catch(async (error) => {
 			this.logger().error('ResourceFetcher: Could not download resource: ' + resource.id, error);
 			await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_ERROR, fetch_error: error.message });
-			completeDownload();
+			await completeDownload();
 		});
 	}