All: Handle deletion of resources that are not linked to any note

CliClient/tests/services_ResourceService.js

@@ -8,6 +8,7 @@ const Note = require('lib/models/Note.js');
 const Tag = require('lib/models/Tag.js');
 const NoteTag = require('lib/models/NoteTag.js');
 const Resource = require('lib/models/Resource.js');
+const NoteResource = require('lib/models/NoteResource.js');
 const ResourceService = require('lib/services/ResourceService.js');
 const fs = require('fs-extra');
 const ArrayUtils = require('lib/ArrayUtils');
@@ -67,6 +68,32 @@ describe('services_ResourceService', function() {
 
 		expect(!!(await Resource.load(resource1.id))).toBe(false);
 		expect(await shim.fsDriver().exists(resourcePath)).toBe(false);
+		expect(!(await NoteResource.all()).length).toBe(true);
+	}));
+
+	it('should not delete resource if still associated with at least one note', asyncTest(async () => {
+		const service = new ResourceService();
+
+		let folder1 = await Folder.save({ title: "folder1" });
+		let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
+		let note2 = await Note.save({ title: 'ma deuxième note', parent_id: folder1.id });
+		note1 = await shim.attachFileToNote(note1, __dirname + '/../tests/support/photo.jpg');
+		let resource1 = (await Resource.all())[0];
+		const resourcePath = Resource.fullPath(resource1);
+
+		await service.indexNoteResources();
+
+		await Note.delete(note1.id);
+		
+		await service.indexNoteResources();
+		
+		await Note.save({ id: note2.id, body: Resource.markdownTag(resource1) });
+
+		await service.indexNoteResources();
+
+		await service.deleteOrphanResources(0);
+
+		expect(!!(await Resource.load(resource1.id))).toBe(true);
 	}));
 
 });

README.md

@@ -4,7 +4,7 @@ Joplin is a free, open source note taking and to-do application, which can handl
 
 Notes exported from Evernote via .enex files [can be imported](#importing) into Joplin, including the formatted content (which is converted to Markdown), resources (images, attachments, etc.) and complete metadata (geolocation, updated time, created time, etc.). Plain Markdown files can also be imported.
 
-The notes can be [synchronised](#synchronisation) with various targets including [Nextcloud](https://nextcloud.com/), the file system (for example with a network directory) or with Microsoft OneDrive. When synchronising the notes, notebooks, tags and other metadata are saved to plain text files which can be easily inspected, backed up and moved around.
+The notes can be [synchronised](#synchronisation) with various cloud services including [Nextcloud](https://nextcloud.com/), the file system (for example with a network directory) or with Microsoft OneDrive. When synchronising the notes, notebooks, tags and other metadata are saved to plain text files which can be easily inspected, backed up and moved around.
 
 The UI of the terminal client is built on top of the great [terminal-kit](https://github.com/cronvel/terminal-kit) library, the desktop client using [Electron](https://electronjs.org/), and the Android client front end is done using [React Native](https://facebook.github.io/react-native/).
 

ReactNativeClient/lib/database.js

@@ -3,14 +3,13 @@ const { promiseChain } = require('lib/promise-utils.js');
 const { Logger } = require('lib/logger.js');
 const { time } = require('lib/time-utils.js');
 const { sprintf } = require('sprintf-js');
+const Mutex = require('async-mutex').Mutex;
 
 class Database {
 
 	constructor(driver) {
 		this.debugMode_ = false;
 		this.driver_ = driver;
-		this.inTransaction_ = false;
-
 		this.logger_ = new Logger();
 		this.logExcludedQueryTypes_ = [];
 	}
@@ -113,29 +112,20 @@ class Database {
 			return;
 		}
 
-		// There can be only one transaction running at a time so queue
+		// There can be only one transaction running at a time so use a	mutex
-		// any new transaction here.
+		const release = await Database.batchTransactionMutex_.acquire();
-		if (this.inTransaction_) {
+
-			while (true) {
+		try {
-				await time.msleep(100);
+			queries.splice(0, 0, 'BEGIN TRANSACTION');
-				if (!this.inTransaction_) {
+			queries.push('COMMIT'); // Note: ROLLBACK is currently not supported
-					this.inTransaction_ = true;
+
-					break;
+			for (let i = 0; i < queries.length; i++) {
-				}
+				let query = this.wrapQuery(queries[i]);
+				await this.exec(query.sql, query.params);
 			}
+		} finally {
+			release();
 		}
-
-		this.inTransaction_ = true;
-
-		queries.splice(0, 0, 'BEGIN TRANSACTION');
-		queries.push('COMMIT'); // Note: ROLLBACK is currently not supported
-
-		for (let i = 0; i < queries.length; i++) {
-			let query = this.wrapQuery(queries[i]);
-			await this.exec(query.sql, query.params);
-		}
-
-		this.inTransaction_ = false;
 	}
 
 	static enumId(type, s) {
@@ -310,6 +300,8 @@ class Database {
 
 }
 
+Database.batchTransactionMutex_ = new Mutex();
+
 Database.TYPE_UNKNOWN = 0;
 Database.TYPE_INT = 1;
 Database.TYPE_TEXT = 2;

ReactNativeClient/lib/joplin-database.js

@@ -336,6 +336,9 @@ class JoplinDatabase extends Database {
 			}
 
 			if (targetVersion == 11) {
+				// This trick was needed because Electron Builder incorrectly released a dev branch containing v10 as it was
+				// still being developed, and the db schema was not final at that time. So this v11 was created to
+				// make sure any invalid db schema that was accidentally created was deleted and recreated.
 				queries.push('DROP TABLE item_changes');
 				queries.push('DROP TABLE note_resources');
 				upgradeVersion10();

ReactNativeClient/lib/models/NoteResource.js

@@ -34,6 +34,16 @@ class NoteResource extends BaseModel {
 		await this.db().transactionExecBatch(queries);
 	}
 
+	static async addOrphanedResources() {
+		const missingResources = await this.db().selectAll('SELECT id FROM resources WHERE id NOT IN (SELECT DISTINCT resource_id FROM note_resources)');
+		const queries = [];
+		for (let i = 0; i < missingResources.length; i++) {
+			const id = missingResources[i];
+			queries.push({ sql: 'INSERT INTO note_resources (note_id, resource_id, is_associated, last_seen_time) VALUES (?, ?, ?, ?)', params: ["", id, 0, Date.now()] });			
+		}
+		await this.db().transactionExecBatch(queries);
+	}
+
 	static async remove(noteId) {
 		await this.db().exec({ sql: 'UPDATE note_resources SET is_associated = 0 WHERE note_id = ?', params: [noteId] });
 	}
@@ -41,10 +51,20 @@ class NoteResource extends BaseModel {
 	static async orphanResources(expiryDelay = null) {
 		if (expiryDelay === null) expiryDelay = 1000 * 60 * 60 * 24;
 		const cutOffTime = Date.now() - expiryDelay;
-		const output = await this.modelSelectAll('SELECT DISTINCT resource_id FROM note_resources WHERE is_associated = 0 AND last_seen_time < ?', [cutOffTime]);
+		const output = await this.modelSelectAll(`
+			SELECT resource_id, sum(is_associated)
+			FROM note_resources
+			GROUP BY resource_id
+			HAVING sum(is_associated) <= 0
+			AND last_seen_time < ?
+		`, [cutOffTime]);
 		return output.map(r => r.resource_id);
 	}
 
+	static async deleteByResource(resourceId) {
+		await this.db().exec('DELETE FROM note_resources WHERE resource_id = ?', [resourceId]);
+	}
+
 }
 
 module.exports = NoteResource;

ReactNativeClient/lib/models/Resource.js

@@ -1,5 +1,6 @@
 const BaseModel = require('lib/BaseModel.js');
 const BaseItem = require('lib/models/BaseItem.js');
+const NoteResource = require('lib/models/NoteResource.js');
 const Setting = require('lib/models/Setting.js');
 const ArrayUtils = require('lib/ArrayUtils.js');
 const pathUtils = require('lib/path-utils.js');
@@ -152,7 +153,8 @@ class Resource extends BaseItem {
 			const resource = await Resource.load(id);
 			const path = Resource.fullPath(resource);
 			await this.fsDriver().remove(path);
-			await super.batchDelete([id], options)
+			await super.batchDelete([id], options);
+			await NoteResource.deleteByResource(id); // Clean up note/resource relationships
 		}
 	}
 

ReactNativeClient/lib/services/ResourceService.js

@@ -62,6 +62,8 @@ class ResourceService extends BaseService {
 			await ItemChange.db().exec('DELETE FROM item_changes WHERE id <= ?', [lastId]);
 		}
 
+		await NoteResource.addOrphanedResources();
+
 		this.logger().info('ResourceService::indexNoteResources: Completed');
 	}
 

docs/index.html

@@ -205,7 +205,7 @@
 <div class="content">
 <p>Joplin is a free, open source note taking and to-do application, which can handle a large number of notes organised into notebooks. The notes are searchable, can be copied, tagged and modified either from the applications directly or from your own text editor. The notes are in <a href="#markdown">Markdown format</a>.</p>
 <p>Notes exported from Evernote via .enex files <a href="#importing">can be imported</a> into Joplin, including the formatted content (which is converted to Markdown), resources (images, attachments, etc.) and complete metadata (geolocation, updated time, created time, etc.). Plain Markdown files can also be imported.</p>
-<p>The notes can be <a href="#synchronisation">synchronised</a> with various targets including <a href="https://nextcloud.com/">Nextcloud</a>, the file system (for example with a network directory) or with Microsoft OneDrive. When synchronising the notes, notebooks, tags and other metadata are saved to plain text files which can be easily inspected, backed up and moved around.</p>
+<p>The notes can be <a href="#synchronisation">synchronised</a> with various cloud services including <a href="https://nextcloud.com/">Nextcloud</a>, the file system (for example with a network directory) or with Microsoft OneDrive. When synchronising the notes, notebooks, tags and other metadata are saved to plain text files which can be easily inspected, backed up and moved around.</p>
 <p>The UI of the terminal client is built on top of the great <a href="https://github.com/cronvel/terminal-kit">terminal-kit</a> library, the desktop client using <a href="https://electronjs.org/">Electron</a>, and the Android client front end is done using <a href="https://facebook.github.io/react-native/">React Native</a>.</p>
 <div class="top-screenshot"><img src="https://raw.githubusercontent.com/laurent22/joplin/master/docs/images/AllClients.jpg" style="max-width: 100%; max-height: 35em;"></div>
 
@@ -409,14 +409,14 @@ $$
 <td><img src="https://raw.githubusercontent.com/stevenrskelton/flag-icon/master/png/16/country-4x3/hr.png" alt=""></td>
 <td>Croatian</td>
 <td><a href="https://github.com/laurent22/joplin/blob/master/CliClient/locales/hr_HR.po">hr_HR</a></td>
-<td>Hrvoje Mandić <a href="&#109;&#97;&#x69;&#108;&#x74;&#111;&#x3a;&#x74;&#114;&#98;&#x75;&#x68;&#111;&#109;&#64;&#110;&#101;&#116;&#46;&#104;&#114;">&#x74;&#114;&#98;&#x75;&#x68;&#111;&#109;&#64;&#110;&#101;&#116;&#46;&#104;&#114;</a></td>
+<td>Hrvoje Mandić <a href="&#x6d;&#x61;&#x69;&#108;&#116;&#111;&#x3a;&#116;&#114;&#x62;&#117;&#104;&#x6f;&#109;&#x40;&#x6e;&#101;&#116;&#46;&#x68;&#x72;">&#116;&#114;&#x62;&#117;&#104;&#x6f;&#109;&#x40;&#x6e;&#101;&#116;&#46;&#x68;&#x72;</a></td>
 <td>64%</td>
 </tr>
 <tr>
 <td><img src="https://raw.githubusercontent.com/stevenrskelton/flag-icon/master/png/16/country-4x3/de.png" alt=""></td>
 <td>Deutsch</td>
 <td><a href="https://github.com/laurent22/joplin/blob/master/CliClient/locales/de_DE.po">de_DE</a></td>
-<td>Tobias Grasse <a href="&#109;&#x61;&#x69;&#108;&#x74;&#x6f;&#x3a;&#x6d;&#97;&#x69;&#x6c;&#x40;&#116;&#111;&#98;&#x69;&#x61;&#x73;&#x2d;&#x67;&#x72;&#97;&#x73;&#115;&#101;&#46;&#110;&#x65;&#116;">&#x6d;&#97;&#x69;&#x6c;&#x40;&#116;&#111;&#98;&#x69;&#x61;&#x73;&#x2d;&#x67;&#x72;&#97;&#x73;&#115;&#101;&#46;&#110;&#x65;&#116;</a></td>
+<td>Tobias Grasse <a href="&#x6d;&#x61;&#x69;&#x6c;&#x74;&#x6f;&#x3a;&#109;&#97;&#x69;&#x6c;&#x40;&#116;&#111;&#x62;&#x69;&#97;&#x73;&#45;&#103;&#x72;&#97;&#x73;&#x73;&#101;&#x2e;&#x6e;&#x65;&#116;">&#109;&#97;&#x69;&#x6c;&#x40;&#116;&#111;&#x62;&#x69;&#97;&#x73;&#45;&#103;&#x72;&#97;&#x73;&#x73;&#101;&#x2e;&#x6e;&#x65;&#116;</a></td>
 <td>99%</td>
 </tr>
 <tr>
@@ -430,7 +430,7 @@ $$
 <td><img src="https://raw.githubusercontent.com/stevenrskelton/flag-icon/master/png/16/country-4x3/es.png" alt=""></td>
 <td>Español</td>
 <td><a href="https://github.com/laurent22/joplin/blob/master/CliClient/locales/es_ES.po">es_ES</a></td>
-<td>Fernando Martín <a href="&#x6d;&#x61;&#105;&#108;&#x74;&#111;&#58;&#x66;&#x40;&#x6d;&#x72;&#116;&#110;&#x2e;&#101;&#x73;">&#x66;&#x40;&#x6d;&#x72;&#116;&#110;&#x2e;&#101;&#x73;</a></td>
+<td>Fernando Martín <a href="&#109;&#x61;&#105;&#x6c;&#116;&#x6f;&#x3a;&#x66;&#x40;&#x6d;&#114;&#116;&#x6e;&#x2e;&#x65;&#115;">&#x66;&#x40;&#x6d;&#114;&#116;&#x6e;&#x2e;&#x65;&#115;</a></td>
 <td>99%</td>
 </tr>
 <tr>
@@ -458,21 +458,21 @@ $$
 <td><img src="https://raw.githubusercontent.com/stevenrskelton/flag-icon/master/png/16/country-4x3/br.png" alt=""></td>
 <td>Português (Brasil)</td>
 <td><a href="https://github.com/laurent22/joplin/blob/master/CliClient/locales/pt_BR.po">pt_BR</a></td>
-<td>Renato Nunes Bastos <a href="&#x6d;&#x61;&#105;&#108;&#x74;&#111;&#x3a;&#114;&#x6e;&#x62;&#97;&#x73;&#x74;&#111;&#x73;&#x40;&#103;&#x6d;&#97;&#105;&#x6c;&#x2e;&#99;&#111;&#x6d;">&#114;&#x6e;&#x62;&#97;&#x73;&#x74;&#111;&#x73;&#x40;&#103;&#x6d;&#97;&#105;&#x6c;&#x2e;&#99;&#111;&#x6d;</a></td>
+<td>Renato Nunes Bastos <a href="&#x6d;&#97;&#105;&#108;&#x74;&#111;&#58;&#x72;&#110;&#x62;&#97;&#115;&#x74;&#x6f;&#115;&#64;&#x67;&#x6d;&#x61;&#105;&#108;&#46;&#99;&#x6f;&#109;">&#x72;&#110;&#x62;&#97;&#115;&#x74;&#x6f;&#115;&#64;&#x67;&#x6d;&#x61;&#105;&#108;&#46;&#99;&#x6f;&#109;</a></td>
 <td>98%</td>
 </tr>
 <tr>
 <td><img src="https://raw.githubusercontent.com/stevenrskelton/flag-icon/master/png/16/country-4x3/ru.png" alt=""></td>
 <td>Русский</td>
 <td><a href="https://github.com/laurent22/joplin/blob/master/CliClient/locales/ru_RU.po">ru_RU</a></td>
-<td>Artyom Karlov <a href="&#109;&#97;&#x69;&#x6c;&#116;&#x6f;&#58;&#x61;&#x72;&#x74;&#121;&#x6f;&#x6d;&#46;&#x6b;&#x61;&#x72;&#x6c;&#111;&#x76;&#64;&#x67;&#109;&#97;&#105;&#x6c;&#46;&#99;&#x6f;&#109;">&#x61;&#x72;&#x74;&#121;&#x6f;&#x6d;&#46;&#x6b;&#x61;&#x72;&#x6c;&#111;&#x76;&#64;&#x67;&#109;&#97;&#105;&#x6c;&#46;&#99;&#x6f;&#109;</a></td>
+<td>Artyom Karlov <a href="&#109;&#97;&#105;&#x6c;&#116;&#111;&#58;&#x61;&#114;&#x74;&#121;&#x6f;&#109;&#x2e;&#x6b;&#x61;&#x72;&#108;&#111;&#118;&#x40;&#103;&#109;&#x61;&#105;&#108;&#46;&#99;&#x6f;&#109;">&#x61;&#114;&#x74;&#121;&#x6f;&#109;&#x2e;&#x6b;&#x61;&#x72;&#108;&#111;&#118;&#x40;&#103;&#109;&#x61;&#105;&#108;&#46;&#99;&#x6f;&#109;</a></td>
 <td>99%</td>
 </tr>
 <tr>
 <td><img src="https://raw.githubusercontent.com/stevenrskelton/flag-icon/master/png/16/country-4x3/cn.png" alt=""></td>
 <td>中文 (简体)</td>
 <td><a href="https://github.com/laurent22/joplin/blob/master/CliClient/locales/zh_CN.po">zh_CN</a></td>
-<td>RCJacH <a href="&#109;&#97;&#x69;&#108;&#116;&#111;&#x3a;&#82;&#x43;&#x4a;&#97;&#x63;&#x48;&#x40;&#x6f;&#x75;&#x74;&#x6c;&#111;&#x6f;&#x6b;&#x2e;&#99;&#x6f;&#x6d;">&#82;&#x43;&#x4a;&#97;&#x63;&#x48;&#x40;&#x6f;&#x75;&#x74;&#x6c;&#111;&#x6f;&#x6b;&#x2e;&#99;&#x6f;&#x6d;</a></td>
+<td>RCJacH <a href="&#x6d;&#97;&#x69;&#x6c;&#x74;&#x6f;&#x3a;&#x52;&#67;&#74;&#x61;&#99;&#72;&#64;&#111;&#x75;&#x74;&#x6c;&#x6f;&#x6f;&#107;&#46;&#x63;&#x6f;&#109;">&#x52;&#67;&#74;&#x61;&#99;&#72;&#64;&#111;&#x75;&#x74;&#x6c;&#x6f;&#x6f;&#107;&#46;&#x63;&#x6f;&#109;</a></td>
 <td>66%</td>
 </tr>
 <tr>