All: Resolves #1481: New: Allow downloading attachments on demand or automatically (#1527)

* Allow downloading resources automatically, on demand, or when loading note

* Make needToBeFetched calls to return the right number of resources

* All: Improved handling of resource downloading and decryption

* Desktop: Click on resource to download it (and, optionally, to decrypt it)

* Desktop: Better handling of resource state (not downloaded, downloading, encrypted) in front end

* Renamed setting to sync.resourceDownloadMode

* Download resources when changing setting

* tweaks

* removed duplicate cs

* Better report resource download progress

* Make sure resource cache is properly cleared when needed

* Also handle manual download for non-image resources

* More improvements to logic when downloading and decrypting resources

ReactNativeClient/lib/services/DecryptionWorker.js

@@ -2,6 +2,7 @@ const BaseItem = require('lib/models/BaseItem');
 const Resource = require('lib/models/Resource');
 const ResourceService = require('lib/services/ResourceService');
 const { Logger } = require('lib/logger.js');
+const EventEmitter = require('events');
 
 class DecryptionWorker {
 
@@ -14,6 +15,7 @@ class DecryptionWorker {
 		};
 
 		this.scheduleId_ = null;
+		this.eventEmitter_ = new EventEmitter();
 	}
 
 	setLogger(l) {
@@ -24,6 +26,14 @@ class DecryptionWorker {
 		return this.logger_;
 	}
 
+	on(eventName, callback) {
+		return this.eventEmitter_.on(eventName, callback);
+	}
+
+	off(eventName, callback) {
+		return this.eventEmitter_.removeListener(eventName, callback);
+	}
+
 	static instance() {
 		if (this.instance_) return this.instance_;
 		this.instance_ = new DecryptionWorker();
@@ -85,14 +95,6 @@ class DecryptionWorker {
 
 					const ItemClass = BaseItem.itemClass(item);
 
-					if (item.type_ === Resource.modelType()) {
-						const ls = await Resource.localState(item);
-						if (ls.fetch_status !== Resource.FETCH_STATUS_DONE) {
-							excludedIds.push(item.id);
-							continue;
-						}
-					}
-
 					this.dispatchReport({
 						itemIndex: i,
 						itemCount: items.length,
@@ -101,10 +103,21 @@ class DecryptionWorker {
 					// Don't log in production as it results in many messages when importing many items
 					// this.logger().info('DecryptionWorker: decrypting: ' + item.id + ' (' + ItemClass.tableName() + ')');
 					try {
-						await ItemClass.decrypt(item);
+						const decryptedItem = await ItemClass.decrypt(item);
+
+						if (decryptedItem.type_ === Resource.modelType() && !!decryptedItem.encryption_blob_encrypted) {
+							// itemsThatNeedDecryption() will return the resource again if the blob has not been decrypted,
+							// but that will result in an infinite loop if the blob simply has not been downloaded yet.
+							// So skip the ID for now, and the service will try to decrypt the blob again the next time. 
+							excludedIds.push(decryptedItem.id);
+						}
+
+						if (decryptedItem.type_ === Resource.modelType() && !decryptedItem.encryption_blob_encrypted) {
+							this.eventEmitter_.emit('resourceDecrypted', { id: decryptedItem.id });
+						}
 					} catch (error) {
 						excludedIds.push(item.id);
-						
+
 						if (error.code === 'masterKeyNotLoaded' && options.masterKeyNotLoadedHandler === 'dispatch') {
 							if (notLoadedMasterKeyDisptaches.indexOf(error.masterKeyId) < 0) {
 								this.dispatch({
@@ -139,9 +152,16 @@ class DecryptionWorker {
 
 		this.logger().info('DecryptionWorker: completed decryption.');
 
+		const downloadedButEncryptedBlobCount = await Resource.downloadedButEncryptedBlobCount();
+
 		this.dispatchReport({ state: 'idle' });
 
 		this.state_ = 'idle';
+
+		if (downloadedButEncryptedBlobCount) {
+			this.logger().info('DecryptionWorker: Some resources have been downloaded but are not decrypted yet. Scheduling another decryption. Resource count: ' + downloadedButEncryptedBlobCount);
+			this.scheduleStart();
+		}
 	}
 
 }

ReactNativeClient/lib/services/ResourceFetcher.js

@@ -1,4 +1,5 @@
 const Resource = require('lib/models/Resource');
+const Setting = require('lib/models/Setting');
 const BaseService = require('lib/services/BaseService');
 const ResourceService = require('lib/services/ResourceService');
 const BaseSyncTarget = require('lib/BaseSyncTarget');
@@ -63,23 +64,32 @@ class ResourceFetcher extends BaseService {
 	}
 
 	updateReport() {
-		if (this.updateReportIID_) return;
-
-		this.updateReportIID_ = setTimeout(async () => {
-			const toFetchCount = await Resource.needToBeFetchedCount();
-			this.dispatch({
-				type: 'RESOURCE_FETCHER_SET',
-				toFetchCount: toFetchCount,
-			});
-			this.updateReportIID_ = null;
-		}, 2000);
+		const fetchingCount = Object.keys(this.fetchingItems_).length;
+		this.dispatch({
+			type: 'RESOURCE_FETCHER_SET',
+			fetchingCount: fetchingCount,
+			toFetchCount: fetchingCount + this.queue_.length,
+		});
 	}
 
-	queueDownload(resourceId, priority = null) {
+	async markForDownload(resourceIds) {
+		if (!Array.isArray(resourceIds)) resourceIds = [resourceIds];
+
+		for (const id of resourceIds) {
+			await Resource.markForDownload(id);
+		}
+
+		for (const id of resourceIds) {
+			this.queueDownload_(id, 'high');
+		}
+	}
+
+	queueDownload_(resourceId, priority = null) {
 		if (priority === null) priority = 'normal';
 
 		const index = this.queuedItemIndex_(resourceId);
 		if (index >= 0) return false;
+		if (this.fetchingItems_[resourceId]) return false;
 
 		const item = { id: resourceId };
 
@@ -99,6 +109,8 @@ class ResourceFetcher extends BaseService {
 		if (this.fetchingItems_[resourceId]) return;
 		this.fetchingItems_[resourceId] = true;
 
+		this.updateReport();
+
 		const resource = await Resource.load(resourceId);
 		const localState = await Resource.localState(resource);
 
@@ -118,7 +130,7 @@ class ResourceFetcher extends BaseService {
 			// might still be encrypted and the caller usually can't do much with this. In particular
 			// the note being displayed will refresh the resource images but since they are still
 			// encrypted it's not useful. Probably, the views should listen to DecryptionWorker events instead.
-			if (emitDownloadComplete) this.eventEmitter_.emit('downloadComplete', { id: resource.id });
+			if (resource && emitDownloadComplete) this.eventEmitter_.emit('downloadComplete', { id: resource.id, encrypted: !!resource.encryption_blob_encrypted });
 			this.updateReport();
 		}
 
@@ -137,7 +149,7 @@ class ResourceFetcher extends BaseService {
 
 		this.fetchingItems_[resourceId] = resource;
 
-		const localResourceContentPath = Resource.fullPath(resource);
+		const localResourceContentPath = Resource.fullPath(resource, !!resource.encryption_blob_encrypted);
 		const remoteResourceContentPath = this.resourceDirName_ + "/" + resource.id;
 
 		await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_STARTED });
@@ -146,6 +158,8 @@ class ResourceFetcher extends BaseService {
 
 		this.logger().debug('ResourceFetcher: Downloading resource: ' + resource.id);
 
+		this.eventEmitter_.emit('downloadStarted', { id: resource.id })
+
 		fileApi.get(remoteResourceContentPath, { path: localResourceContentPath, target: "file" }).then(async () => {
 			await Resource.setLocalState(resource, { fetch_status: Resource.FETCH_STATUS_DONE });
 			this.logger().debug('ResourceFetcher: Resource downloaded: ' + resource.id);
@@ -184,10 +198,12 @@ class ResourceFetcher extends BaseService {
 		if (this.addingResources_) return;
 		this.addingResources_ = true;
 
+		this.logger().info('ResourceFetcher: Auto-add resources: Mode: ' + Setting.value('sync.resourceDownloadMode'));
+
 		let count = 0;
-		const resources = await Resource.needToBeFetched(limit);
+		const resources = await Resource.needToBeFetched(Setting.value('sync.resourceDownloadMode'), limit);
 		for (let i = 0; i < resources.length; i++) {
-			const added = this.queueDownload(resources[i].id);
+			const added = this.queueDownload_(resources[i].id);
 			if (added) count++;
 		}