All: Save size of a resource to the database; and added mechanism to run non-database migrations

ReactNativeClient/lib/BaseApplication.js

@@ -39,6 +39,7 @@ const RevisionService = require('lib/services/RevisionService');
 const DecryptionWorker = require('lib/services/DecryptionWorker');
 const BaseService = require('lib/services/BaseService');
 const SearchEngine = require('lib/services/SearchEngine');
+const MigrationService = require('lib/services/MigrationService');
 
 SyncTargetRegistry.addClass(SyncTargetFilesystem);
 SyncTargetRegistry.addClass(SyncTargetOneDrive);
@@ -606,6 +607,8 @@ class BaseApplication {
 		if (!currentFolder) currentFolder = await Folder.defaultFolder();
 		Setting.setValue('activeFolderId', currentFolder ? currentFolder.id : '');
 
+		await MigrationService.instance().run();
+
 		return argv;
 	}
 

ReactNativeClient/lib/BaseModel.js

@@ -540,6 +540,7 @@ BaseModel.typeEnum_ = [
 	['TYPE_NOTE_RESOURCE', 11],
 	['TYPE_RESOURCE_LOCAL_STATE', 12],
 	['TYPE_REVISION', 13],
+	['TYPE_MIGRATION', 14],
 ];
 
 for (let i = 0; i < BaseModel.typeEnum_.length; i++) {

ReactNativeClient/lib/components/screens/note.js

@@ -424,6 +424,9 @@ class NoteScreenComponent extends BaseScreenComponent {
 			return;
 		}
 
+		const fileStat = await shim.fsDriver().stat(targetPath);
+		resource.size = fileStat.size;
+
 		await Resource.save(resource, { isNew: true });
 
 		const resourceTag = Resource.markdownTag(resource);

ReactNativeClient/lib/import-enex.js

@@ -408,12 +408,18 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
 					debugTemp.data = debugTemp.data ? debugTemp.data.substr(0, 32) + '...' : debugTemp.data;
 					importOptions.onError(new Error('This resource was not added because it has no ID or no content: ' + JSON.stringify(debugTemp)));
 				} else {
+					let size = 0;
+					if (decodedData) {
+						size = 'byteLength' in decodedData ? decodedData.byteLength : decodedData.length;
+					}
+
 					let r = {
 						id: resourceId,
 						data: decodedData,
 						mime: noteResource.mime,
 						title: noteResource.filename ? noteResource.filename : '',
 						filename: noteResource.filename ? noteResource.filename : '',
+						size: size,
 					};
 
 					note.resources.push(r);

ReactNativeClient/lib/joplin-database.js

@@ -263,7 +263,7 @@ class JoplinDatabase extends Database {
 		// must be set in the synchronizer too.
 
 		// Note: v16 and v17 don't do anything. They were used to debug an issue.
-		const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
+		const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20];
 
 		let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
 
@@ -544,6 +544,22 @@ class JoplinDatabase extends Database {
 				queries.push('ALTER TABLE item_changes ADD COLUMN before_change_item TEXT NOT NULL DEFAULT ""');
 			}
 
+			if (targetVersion == 20) {
+				const newTableSql = `
+					CREATE TABLE migrations (
+						id INTEGER PRIMARY KEY,
+						number INTEGER NOT NULL,
+						updated_time INT NOT NULL,
+						created_time INT NOT NULL
+					);
+				`;
+				queries.push(this.sqlStringToLines(newTableSql)[0]);
+
+				const timestamp = Date.now();
+				queries.push('ALTER TABLE resources ADD COLUMN `size` INT NOT NULL DEFAULT -1');
+				queries.push({ sql: 'INSERT INTO migrations (number, created_time, updated_time) VALUES (20, ?, ?)', params: [timestamp, timestamp] });
+			}
+
 			queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });
 
 			try {

ReactNativeClient/lib/migrations/20.js

@@ -0,0 +1,25 @@
+const Resource = require('lib/models/Resource');
+const Setting = require('lib/models/Setting');
+const { shim } = require('lib/shim');
+const { reg } = require('lib/registry.js');
+
+const script = {};
+
+script.exec = async function() {
+	const stats = await shim.fsDriver().readDirStats(Setting.value('resourceDir'));
+
+	const queries = [];
+	for (const stat of stats) {
+		const resourceId = Resource.pathToId(stat.path);
+		queries.push({ sql: 'UPDATE resources SET `size` = ? WHERE id = ?', params: [stat.size, resourceId] });
+
+		if (queries.length >= 1000) {
+			await reg.db().transactionExecBatch(queries);
+			queries = [];
+		}
+	}
+
+	await reg.db().transactionExecBatch(queries);
+}
+
+module.exports = script;

ReactNativeClient/lib/models/Migration.js

@@ -0,0 +1,27 @@
+const BaseModel = require('lib/BaseModel.js');
+
+const migrationScripts = {
+	20: require('lib/migrations/20.js'),
+};
+
+class Migration extends BaseModel {
+
+	static tableName() {
+		return 'migrations';
+	}
+
+	static modelType() {
+		return BaseModel.TYPE_MIGRATION;
+	}
+
+	static migrationsToDo() {
+		return this.modelSelectAll('SELECT * FROM migrations ORDER BY number ASC');
+	}
+
+	static script(number) {
+		return migrationScripts[number];
+	}
+
+}
+
+module.exports = Migration;

ReactNativeClient/lib/services/MigrationService.js

@@ -0,0 +1,37 @@
+const BaseService = require('lib/services/BaseService');
+const Migration = require('lib/models/Migration');
+
+class MigrationService extends BaseService {
+
+	constructor() {
+		super();
+
+	}
+
+	static instance() {
+		if (this.instance_) return this.instance_;
+		this.instance_ = new MigrationService();
+		return this.instance_;
+	}
+
+	async run() {
+		const migrations = await Migration.migrationsToDo();
+
+		for (const migration of migrations) {
+			this.logger().info('Running migration: ' + migration.number);
+
+			const script = Migration.script(migration.number);
+
+			try {
+				await script.exec();
+				await Migration.delete(migration.id);
+			} catch (error) {
+				this.logger().error('Cannot run migration: ' + migration.number, error);
+				break;
+			}
+		}
+	}
+
+}
+
+module.exports = MigrationService;

ReactNativeClient/lib/shim-init-node.js

@@ -152,6 +152,9 @@ function shimInit() {
 			resource = Object.assign({}, resource, defaultProps);
 		}
 
+		const fileStat = await shim.fsDriver().stat(targetPath);
+		resource.size = fileStat.size;
+
 		return await Resource.save(resource, { isNew: true });
 	}
 

ReactNativeClient/lib/shim-init-react.js

@@ -166,9 +166,10 @@ function shimInit() {
 			resource = Object.assign({}, resource, defaultProps);
 		}
 
-		resource = await Resource.save(resource, { isNew: true });
+		const fileStat = await shim.fsDriver().stat(targetPath);
+		resource.size = fileStat.size;
 
-		console.info(resource);
+		resource = await Resource.save(resource, { isNew: true });
 
 		return resource;
 	}

ReactNativeClient/root.js

@@ -76,6 +76,7 @@ SyncTargetRegistry.addClass(SyncTargetFilesystem);
 const FsDriverRN = require('lib/fs-driver-rn.js').FsDriverRN;
 const DecryptionWorker = require('lib/services/DecryptionWorker');
 const EncryptionService = require('lib/services/EncryptionService');
+const MigrationService = require('lib/services/MigrationService');
 
 let storeDispatch = function(action) {};
 
@@ -510,6 +511,8 @@ async function initialize(dispatch) {
 	SearchEngine.instance().setLogger(reg.logger());
 	SearchEngine.instance().scheduleSyncTables();
 
+	await MigrationService.instance().run();
+
 	reg.scheduleSync().then(() => {
 		// Wait for the first sync before updating the notifications, since synchronisation
 		// might change the notifications.