All: Improved: Better handling of items that cannot be decrypted, including those that cause crashes

ReactNativeClient/lib/services/DecryptionWorker.js

@@ -2,6 +2,7 @@ const BaseItem = require('lib/models/BaseItem');
 const MasterKey = require('lib/models/MasterKey');
 const Resource = require('lib/models/Resource');
 const ResourceService = require('lib/services/ResourceService');
+const KvStore = require('lib/services/KvStore');
 const { Logger } = require('lib/logger.js');
 const EventEmitter = require('events');
 
@@ -17,6 +18,8 @@ class DecryptionWorker {
 
 		this.scheduleId_ = null;
 		this.eventEmitter_ = new EventEmitter();
+		this.kvStore_ = null;
+		this.maxDecryptionAttempts_ = 2;
 	}
 
 	setLogger(l) {
@@ -45,11 +48,20 @@ class DecryptionWorker {
 		this.encryptionService_ = v;
 	}
 
+	setKvStore(v) {
+		this.kvStore_ = v;
+	}
+
 	encryptionService() {
 		if (!this.encryptionService_) throw new Error('DecryptionWorker.encryptionService_ is not set!!');
 		return this.encryptionService_;
 	}
 
+	kvStore() {
+		if (!this.kvStore_) throw new Error('DecryptionWorker.kvStore_ is not set!!');
+		return this.kvStore_;
+	}
+
 	async scheduleStart() {
 		if (this.scheduleId_) return;
 
@@ -61,6 +73,23 @@ class DecryptionWorker {
 		}, 1000);
 	}
 
+	async decryptionDisabledItems() {
+		let items = await this.kvStore().searchByPrefix('decrypt:');
+		items = items.filter(item => item.value > this.maxDecryptionAttempts_);
+		items = items.map(item => {
+			const s = item.key.split(':');
+			return {
+				type_: Number(s[1]),
+				id: s[2],
+			};
+		});
+		return items;
+	}
+
+	async clearDisabledItem(typeId, itemId) {
+		await this.kvStore().deleteValue('decrypt:' + typeId + ':' + itemId);
+	}
+
 	dispatchReport(report) {
 		const action = Object.assign({}, report);
 		action.type = 'DECRYPTION_WORKER_SET';
@@ -70,6 +99,7 @@ class DecryptionWorker {
 	async start(options = null) {
 		if (options === null) options = {};
 		if (!('masterKeyNotLoadedHandler' in options)) options.masterKeyNotLoadedHandler = 'throw';
+		if (!('errorHandler' in options)) options.errorHandler = 'log';
 
 		if (this.state_ !== 'idle') {
 			this.logger().info('DecryptionWorker: cannot start because state is "' + this.state_ + '"');
@@ -114,12 +144,27 @@ class DecryptionWorker {
 						itemIndex: i,
 						itemCount: items.length,
 					});
+
+					const counterKey = 'decrypt:' + item.type_ + ':' + item.id;
+
+					const clearDecryptionCounter = async () => {
+						await this.kvStore().deleteValue(counterKey);
+					}
 					
 					// Don't log in production as it results in many messages when importing many items
 					// this.logger().debug('DecryptionWorker: decrypting: ' + item.id + ' (' + ItemClass.tableName() + ')');
 					try {
+						const decryptCounter = await this.kvStore().incValue(counterKey);
+						if (decryptCounter > this.maxDecryptionAttempts_) {
+							this.logger().warn('DecryptionWorker: ' + item.id + ' decryption has failed more than 2 times - skipping it');
+							excludedIds.push(item.id);
+							continue;
+						}
+
 						const decryptedItem = await ItemClass.decrypt(item);
 
+						await clearDecryptionCounter();
+
 						if (decryptedItem.type_ === Resource.modelType() && !!decryptedItem.encryption_blob_encrypted) {
 							// itemsThatNeedDecryption() will return the resource again if the blob has not been decrypted,
 							// but that will result in an infinite loop if the blob simply has not been downloaded yet.
@@ -145,14 +190,20 @@ class DecryptionWorker {
 								});
 								notLoadedMasterKeyDisptaches.push(error.masterKeyId);
 							}
+							await clearDecryptionCounter();
 							continue;
 						}
 
 						if (error.code === 'masterKeyNotLoaded' && options.masterKeyNotLoadedHandler === 'throw') {
+							await clearDecryptionCounter();
 							throw error;
 						}
 
-						this.logger().warn('DecryptionWorker: error for: ' + item.id + ' (' + ItemClass.tableName() + ')', error, item);
+						if (options.errorHandler === 'log') {
+							this.logger().warn('DecryptionWorker: error for: ' + item.id + ' (' + ItemClass.tableName() + ')', error, item);
+						} else {
+							throw error;
+						}
 					}
 				}
 

ReactNativeClient/lib/services/KvStore.js

@@ -29,6 +29,15 @@ class KvStore extends BaseService {
 		throw new Error('Unsupported value type: ' + (typeof value));
 	}
 
+	formatValues_(kvs) {
+		const output = [];
+		for (const kv of kvs) {
+			kv.value = this.formatValue_(kv.value, kv.type);
+			output.push(kv);
+		}
+		return output;
+	}
+
 	formatValue_(value, type) {
 		if (type === KvStore.TYPE_INT) return Number(value);
 		if (type === KvStore.TYPE_TEXT) return value + '';
@@ -50,6 +59,14 @@ class KvStore extends BaseService {
 		await this.db().exec('DELETE FROM key_values WHERE `key` = ?', [key]);
 	}
 
+	async clear() {
+		await this.db().exec('DELETE FROM key_values');
+	}
+
+	async all() {
+		return this.formatValues_(await this.db().selectAll('SELECT * FROM key_values'));
+	}
+
 	// Note: atomicity is done at application level so two difference instances
 	// accessing the db at the same time could mess up the increment.
 	async incValue(key, inc = 1) {
@@ -67,6 +84,11 @@ class KvStore extends BaseService {
 		}		
 	}
 
+	async searchByPrefix(prefix) {
+		let results = await this.db().selectAll('SELECT `key`, `value`, `type` FROM key_values WHERE `key` LIKE ?', [prefix + '%']);
+		return this.formatValues_(results);
+	}
+
 	async countKeys() {
 		const r = await this.db().selectOne('SELECT count(*) as total FROM key_values');
 		return r.total ? r.total : 0;

ReactNativeClient/lib/services/report.js

@@ -3,7 +3,10 @@ const BaseItem = require('lib/models/BaseItem.js');
 const Alarm = require('lib/models/Alarm');
 const Folder = require('lib/models/Folder.js');
 const Note = require('lib/models/Note.js');
+const BaseModel = require('lib/BaseModel.js');
+const DecryptionWorker = require('lib/services/DecryptionWorker');
 const { _ } = require('lib/locale.js');
+const { toTitleCase } = require('lib/string-utils.js');
 
 class ReportService {
 
@@ -116,6 +119,10 @@ class ReportService {
 		if (disabledItems.length) {
 			section = { title: _('Items that cannot be synchronised'), body: [] };
 
+			section.body.push(_('These items will remain on the device but will not be uploaded to the sync target. In order to find these items, either search for the title or the ID (which is displayed in brackets above).'));
+
+			section.body.push('');
+
 			for (let i = 0; i < disabledItems.length; i++) {
 				const row = disabledItems[i];
 				if (row.location === BaseItem.SYNC_ITEM_LOCATION_LOCAL) {
@@ -125,8 +132,25 @@ class ReportService {
 				}
 			}
 
+			sections.push(section);
+		}
+
+		const decryptionDisabledItems = await DecryptionWorker.instance().decryptionDisabledItems();
+
+		if (decryptionDisabledItems.length) {
+			section = { title: _('Items that cannot be decrypted'), body: [], name: 'failedDecryption' };
+
+			section.body.push(_('Joplin failed to decrypt these items multiple times, possibly because they are corrupted or too large. These items will remain on the device but Joplin will no longer attempt to decrypt them.'));
+
 			section.body.push('');
-			section.body.push(_('These items will remain on the device but will not be uploaded to the sync target. In order to find these items, either search for the title or the ID (which is displayed in brackets above).'));
+			
+			for (let i = 0; i < decryptionDisabledItems.length; i++) {
+				const row = decryptionDisabledItems[i];
+				section.body.push({ text: _('%s: %s', toTitleCase(BaseModel.modelTypeToName(row.type_)), row.id), canRetry: true, retryHandler: async () => {
+					await DecryptionWorker.instance().clearDisabledItem(row.type_, row.id);
+					DecryptionWorker.instance().scheduleStart();
+				}});
+			}
 
 			sections.push(section);
 		}