Desktop: Resolves #262: Implement "show all notes" (#2472)

* Implement "show all notes" feature.

* Ensure middleware is completely flushed and shutdown before continuing tests.

.eslintrc.js

@@ -19,7 +19,10 @@ module.exports = {
 		'expect': 'readonly',
 		'describe': 'readonly',
 		'it': 'readonly',
+		'beforeAll': 'readonly',
+		'afterAll': 'readonly',
 		'beforeEach': 'readonly',
+		'afterEach': 'readonly',
 		'jasmine': 'readonly',
 
 		// React Native variables

CliClient/tests/integration_SmartFilters.js

@@ -0,0 +1,117 @@
+/* eslint-disable no-unused-vars */
+require('app-module-path').addPath(__dirname);
+const { setupDatabaseAndSynchronizer, switchClient, asyncTest, TestApp } = require('test-utils.js');
+const Setting = require('lib/models/Setting.js');
+const Folder = require('lib/models/Folder.js');
+const Note = require('lib/models/Note.js');
+const Tag = require('lib/models/Tag.js');
+const { time } = require('lib/time-utils.js');
+const { ALL_NOTES_FILTER_ID } = require('lib/reserved-ids.js');
+
+//
+// The integration tests are to test the integration of the core system, comprising the
+// base application with middleware, reducer and models in response to dispatched events.
+//
+// The general strategy for each integration test is:
+//  - create a starting application state,
+//  - inject the event to be tested
+//  - check the resulting application state
+//
+// In particular, this file contains integration tests for smart filter features.
+//
+
+async function createNTestFolders(n) {
+	let folders = [];
+	for (let i = 0; i < n; i++) {
+		let folder = await Folder.save({ title: 'folder' });
+		folders.push(folder);
+	}
+	return folders;
+}
+
+async function createNTestNotes(n, folder) {
+	let notes = [];
+	for (let i = 0; i < n; i++) {
+		let note = await Note.save({ title: 'note', parent_id: folder.id, is_conflict: 0 });
+		notes.push(note);
+	}
+	return notes;
+}
+
+async function createNTestTags(n) {
+	let tags = [];
+	for (let i = 0; i < n; i++) {
+		let tag = await Tag.save({ title: 'tag' });
+		tags.push(tag);
+	}
+	return tags;
+}
+
+// use this until Javascript arr.flat() function works in Travis
+function flatten(arr) {
+	return (arr.reduce((acc, val) => acc.concat(val), []));
+}
+
+let testApp = null;
+
+describe('integration_SmartFilters', function() {
+
+	beforeEach(async (done) => {
+		testApp = new TestApp();
+		await testApp.start(['--no-welcome']);
+		done();
+	});
+
+	afterEach(async (done) => {
+		if (testApp !== null) await testApp.destroy();
+		testApp = null;
+		done();
+	});
+
+	it('should show notes in a folder', asyncTest(async () => {
+		let folders = await createNTestFolders(2);
+		let notes = [];
+		for (let i = 0; i < folders.length; i++) {
+			notes.push(await createNTestNotes(3, folders[i]));
+		}
+
+		testApp.dispatch({
+			type: 'FOLDER_SELECT',
+			id: folders[1].id,
+		});
+		await time.msleep(100);
+
+		let state = testApp.store().getState();
+
+		expect(state.notesParentType).toEqual('Folder');
+		expect(state.selectedFolderId).toEqual(folders[1].id);
+
+		let expectedNoteIds = notes[1].map(n => n.id).sort();
+		let noteIds = state.notes.map(n => n.id).sort();
+		expect(noteIds).toEqual(expectedNoteIds);
+	}));
+
+	it('should show all notes', asyncTest(async () => {
+		let folders = await createNTestFolders(2);
+		let notes = [];
+		for (let i = 0; i < folders.length; i++) {
+			notes.push(await createNTestNotes(3, folders[i]));
+		}
+
+		testApp.dispatch({
+			type: 'SMART_FILTER_SELECT',
+			id: ALL_NOTES_FILTER_ID,
+		});
+		await time.msleep(100);
+
+		let state = testApp.store().getState();
+
+		expect(state.notesParentType).toEqual('SmartFilter');
+		expect(state.selectedSmartFilterId).toEqual(ALL_NOTES_FILTER_ID);
+
+		// let expectedNoteIds = notes.map(n => n.map(o => o.id)).flat().sort();
+		let expectedNoteIds = flatten(notes.map(n => n.map(o => o.id))).sort();
+		let noteIds = state.notes.map(n => n.id).sort();
+		expect(noteIds).toEqual(expectedNoteIds);
+	}));
+});

CliClient/tests/test-utils.js

@@ -3,6 +3,7 @@
 const fs = require('fs-extra');
 const { JoplinDatabase } = require('lib/joplin-database.js');
 const { DatabaseDriverNode } = require('lib/database-driver-node.js');
+const { BaseApplication }= require('lib/BaseApplication.js');
 const BaseModel = require('lib/BaseModel.js');
 const Folder = require('lib/models/Folder.js');
 const Note = require('lib/models/Note.js');
@@ -412,4 +413,49 @@ async function allSyncTargetItemsEncrypted() {
 	return totalCount === encryptedCount;
 }
 
-module.exports = { kvStore, resourceService, allSyncTargetItemsEncrypted, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, asyncTest };
+class TestApp extends BaseApplication {
+	constructor() {
+		super();
+		this.middlewareCalls_ = [];
+	}
+
+	async start(argv) {
+		argv = await super.start(argv);
+		this.initRedux();
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+		Setting.dispatchUpdateAll();
+		await time.msleep(100);
+	}
+
+	async generalMiddleware(store, next, action) {
+		this.middlewareCalls_.push(true);
+		try {
+			await super.generalMiddleware(store, next, action);
+		} finally {
+			this.middlewareCalls_.pop();
+		}
+	}
+
+	async waitForMiddleware_() {
+		return new Promise((resolve) => {
+			const iid = setInterval(() => {
+				if (!this.middlewareCalls_.length) {
+					clearInterval(iid);
+					resolve();
+				}
+			}, 100);
+		});
+	}
+
+	async destroy() {
+		this.deinitRedux();
+		await this.waitForMiddleware_();
+		await super.destroy();
+
+	}
+}
+
+
+module.exports = { kvStore, resourceService, allSyncTargetItemsEncrypted, setupDatabase, revisionService, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync, encryptionService, loadEncryptionMasterKey, fileContentEqual, decryptionWorker, asyncTest, TestApp };
+

ElectronClient/gui/NoteText.jsx

@@ -1621,7 +1621,7 @@ class NoteTextComponent extends React.Component {
 
 	createToolbarItems(note, editorIsVisible) {
 		const toolbarItems = [];
-		if (note && this.state.folder && ['Search', 'Tag'].includes(this.props.notesParentType)) {
+		if (note && this.state.folder && ['Search', 'Tag', 'SmartFilter'].includes(this.props.notesParentType)) {
 			toolbarItems.push({
 				title: _('In: %s', substrWithEllipsis(this.state.folder.title, 0, 16)),
 				iconName: 'fa-book',

ElectronClient/gui/SideBar.jsx

@@ -14,6 +14,7 @@ const Menu = bridge().Menu;
 const MenuItem = bridge().MenuItem;
 const InteropServiceHelper = require('../InteropServiceHelper.js');
 const { substrWithEllipsis } = require('lib/string-utils');
+const { ALL_NOTES_FILTER_ID } = require('lib/reserved-ids');
 
 class SideBarComponent extends React.Component {
 	constructor() {
@@ -89,6 +90,7 @@ class SideBarComponent extends React.Component {
 		this.tagItemsOrder_ = [];
 
 		this.onKeyDown = this.onKeyDown.bind(this);
+		this.onAllNotesClick_ = this.onAllNotesClick_.bind(this);
 
 		this.rootRef = React.createRef();
 
@@ -570,6 +572,9 @@ class SideBarComponent extends React.Component {
 			let isExpanded = this.state[toggleKey];
 			toggleIcon = <i className={`fa ${isExpanded ? 'fa-chevron-down' : 'fa-chevron-left'}`} style={{ fontSize: style.fontSize * 0.75, marginRight: 12, marginLeft: 5, marginTop: style.fontSize * 0.125 }}></i>;
 		}
+		if (extraProps.selected) {
+			style.backgroundColor =this.style().listItemSelected.backgroundColor;
+		}
 
 		const ref = this.anchorItemRef('headers', key);
 
@@ -696,6 +701,13 @@ class SideBarComponent extends React.Component {
 		}
 	}
 
+	onAllNotesClick_() {
+		this.props.dispatch({
+			type: 'SMART_FILTER_SELECT',
+			id: ALL_NOTES_FILTER_ID,
+		});
+	}
+
 	synchronizeButton(type) {
 		const style = Object.assign({}, this.style().button, { marginBottom: 5 });
 		const iconName = 'fa-refresh';
@@ -732,6 +744,13 @@ class SideBarComponent extends React.Component {
 		});
 
 		let items = [];
+		items.push(
+			this.makeHeader('allNotesHeader', _('All notes'), 'fa-clone', {
+				onClick: this.onAllNotesClick_,
+				selected: this.props.notesParentType === 'SmartFilter' && this.props.selectedSmartFilterId === ALL_NOTES_FILTER_ID,
+			})
+		);
+
 		items.push(
 			this.makeHeader('folderHeader', _('Notebooks'), 'fa-book', {
 				onDrop: this.onFolderDrop_,
@@ -821,6 +840,7 @@ const mapStateToProps = state => {
 		selectedFolderId: state.selectedFolderId,
 		selectedTagId: state.selectedTagId,
 		selectedSearchId: state.selectedSearchId,
+		selectedSmartFilterId: state.selectedSmartFilterId,
 		notesParentType: state.notesParentType,
 		locale: state.settings.locale,
 		theme: state.settings.theme,

ReactNativeClient/lib/BaseApplication.js

@@ -54,6 +54,18 @@ class BaseApplication {
 		this.decryptionWorker_resourceMetadataButNotBlobDecrypted = this.decryptionWorker_resourceMetadataButNotBlobDecrypted.bind(this);
 	}
 
+	async destroy() {
+		await FoldersScreenUtils.cancelTimers();
+		await SearchEngine.instance().cancelTimers();
+		await DecryptionWorker.instance().cancelTimers();
+		await reg.cancelTimers();
+
+		this.logger_ = null;
+		this.dbLogger_ = null;
+		this.eventEmitter_ = null;
+		this.decryptionWorker_resourceMetadataButNotBlobDecrypted = null;
+	}
+
 	logger() {
 		return this.logger_;
 	}
@@ -217,6 +229,9 @@ class BaseApplication {
 		} else if (parentType === 'Search') {
 			parentId = state.selectedSearchId;
 			parentType = BaseModel.TYPE_SEARCH;
+		} else if (parentType === 'SmartFilter') {
+			parentId = state.selectedSmartFilterId;
+			parentType = BaseModel.TYPE_SMART_FILTER;
 		}
 
 		this.logger().debug('Refreshing notes:', parentType, parentId);
@@ -243,6 +258,8 @@ class BaseApplication {
 			} else if (parentType === BaseModel.TYPE_SEARCH) {
 				const search = BaseModel.byId(state.searches, parentId);
 				notes = await SearchEngineUtils.notesForQuery(search.query_pattern);
+			} else if (parentType === BaseModel.TYPE_SMART_FILTER) {
+				notes = await Note.previews(parentId, options);
 			}
 		}
 
@@ -435,6 +452,11 @@ class BaseApplication {
 			refreshNotes = true;
 		}
 
+		if (action.type == 'SMART_FILTER_SELECT') {
+			refreshNotes = true;
+			refreshNotesUseSelectedNoteId = true;
+		}
+
 		if (action.type == 'TAG_SELECT' || action.type === 'TAG_DELETE') {
 			refreshNotes = true;
 		}
@@ -493,7 +515,6 @@ class BaseApplication {
 				await FoldersScreenUtils.scheduleRefreshFolders();
 			}
 		}
-
 		return result;
 	}
 
@@ -515,6 +536,16 @@ class BaseApplication {
 		ResourceFetcher.instance().dispatch = this.store().dispatch;
 	}
 
+	deinitRedux() {
+		this.store_ = null;
+		BaseModel.dispatch = function() {};
+		FoldersScreenUtils.dispatch = function() {};
+		reg.dispatch = function() {};
+		BaseSyncTarget.dispatch = function() {};
+		DecryptionWorker.instance().dispatch = function() {};
+		ResourceFetcher.instance().dispatch = function() {};
+	}
+
 	async readFlagsFromFile(flagPath) {
 		if (!fs.existsSync(flagPath)) return {};
 		let flagContent = fs.readFileSync(flagPath, 'utf8');
@@ -670,7 +701,6 @@ class BaseApplication {
 		Setting.setValue('activeFolderId', currentFolder ? currentFolder.id : '');
 
 		await MigrationService.instance().run();
-
 		return argv;
 	}
 }

ReactNativeClient/lib/BaseModel.js

@@ -552,7 +552,7 @@ class BaseModel {
 	}
 }
 
-BaseModel.typeEnum_ = [['TYPE_NOTE', 1], ['TYPE_FOLDER', 2], ['TYPE_SETTING', 3], ['TYPE_RESOURCE', 4], ['TYPE_TAG', 5], ['TYPE_NOTE_TAG', 6], ['TYPE_SEARCH', 7], ['TYPE_ALARM', 8], ['TYPE_MASTER_KEY', 9], ['TYPE_ITEM_CHANGE', 10], ['TYPE_NOTE_RESOURCE', 11], ['TYPE_RESOURCE_LOCAL_STATE', 12], ['TYPE_REVISION', 13], ['TYPE_MIGRATION', 14]];
+BaseModel.typeEnum_ = [['TYPE_NOTE', 1], ['TYPE_FOLDER', 2], ['TYPE_SETTING', 3], ['TYPE_RESOURCE', 4], ['TYPE_TAG', 5], ['TYPE_NOTE_TAG', 6], ['TYPE_SEARCH', 7], ['TYPE_ALARM', 8], ['TYPE_MASTER_KEY', 9], ['TYPE_ITEM_CHANGE', 10], ['TYPE_NOTE_RESOURCE', 11], ['TYPE_RESOURCE_LOCAL_STATE', 12], ['TYPE_REVISION', 13], ['TYPE_MIGRATION', 14], ['TYPE_SMART_FILTER', 15]];
 
 for (let i = 0; i < BaseModel.typeEnum_.length; i++) {
 	const e = BaseModel.typeEnum_[i];

ReactNativeClient/lib/folders-screen-utils.js

@@ -34,22 +34,40 @@ class FoldersScreenUtils {
 	}
 
 	static async refreshFolders() {
-		const folders = await this.allForDisplay({ includeConflictFolder: true });
+		FoldersScreenUtils.refreshCalls_.push(true);
+		try {
+			const folders = await this.allForDisplay({ includeConflictFolder: true });
 
-		this.dispatch({
-			type: 'FOLDER_UPDATE_ALL',
-			items: folders,
-		});
+			this.dispatch({
+				type: 'FOLDER_UPDATE_ALL',
+				items: folders,
+			});
+		} finally {
+			FoldersScreenUtils.refreshCalls_.pop();
+		}
 	}
 
 	static scheduleRefreshFolders() {
 		if (this.scheduleRefreshFoldersIID_) clearTimeout(this.scheduleRefreshFoldersIID_);
-
 		this.scheduleRefreshFoldersIID_ = setTimeout(() => {
 			this.scheduleRefreshFoldersIID_ = null;
 			this.refreshFolders();
 		}, 1000);
 	}
+
+	static async cancelTimers() {
+		if (this.scheduleRefreshFoldersIID_) clearTimeout(this.scheduleRefreshFoldersIID_);
+		return new Promise((resolve) => {
+			const iid = setInterval(() => {
+				if (!FoldersScreenUtils.refreshCalls_.length) {
+					clearInterval(iid);
+					resolve();
+				}
+			}, 100);
+		});
+	}
 }
 
+FoldersScreenUtils.refreshCalls_ = [];
+
 module.exports = { FoldersScreenUtils };

ReactNativeClient/lib/models/Note.js

@@ -12,6 +12,7 @@ const ArrayUtils = require('lib/ArrayUtils.js');
 const lodash = require('lodash');
 const urlUtils = require('lib/urlUtils.js');
 const { MarkupToHtml } = require('lib/joplin-renderer');
+const { ALL_NOTES_FILTER_ID } = require('lib/reserved-ids');
 
 class Note extends BaseItem {
 	static tableName() {
@@ -275,7 +276,7 @@ class Note extends BaseItem {
 			options.conditions.push('is_conflict = 1');
 		} else {
 			options.conditions.push('is_conflict = 0');
-			if (parentId) {
+			if (parentId && parentId !== ALL_NOTES_FILTER_ID) {
 				options.conditions.push('parent_id = ?');
 				options.conditionsParams.push(parentId);
 			}

ReactNativeClient/lib/models/SmartFilter.js

@@ -0,0 +1,13 @@
+const BaseModel = require('lib/BaseModel.js');
+
+class SmartFilter extends BaseModel {
+	static tableName() {
+		throw new Error('Not using database');
+	}
+
+	static modelType() {
+		return BaseModel.TYPE_SMART_FILTER;
+	}
+}
+
+module.exports = SmartFilter;

ReactNativeClient/lib/reducer.js

@@ -399,6 +399,12 @@ const reducer = (state = defaultState, action) => {
 			newState.selectedNoteIds = newState.notes.map(n => n.id);
 			break;
 
+		case 'SMART_FILTER_SELECT':
+			newState = Object.assign({}, state);
+			newState.notesParentType = 'SmartFilter';
+			newState.selectedSmartFilterId = action.id;
+			break;
+
 		case 'FOLDER_SELECT':
 			newState = changeSelectedFolder(state, action, { clearSelectedNoteIds: true });
 			break;

ReactNativeClient/lib/registry.js

@@ -52,7 +52,7 @@ reg.syncTarget = (syncTargetId = null) => {
 	return target;
 };
 
-reg.scheduleSync = async (delay = null, syncOptions = null) => {
+reg.scheduleSync_ = async (delay = null, syncOptions = null) => {
 	if (delay === null) delay = 1000 * 10;
 	if (syncOptions === null) syncOptions = {};
 
@@ -152,6 +152,15 @@ reg.scheduleSync = async (delay = null, syncOptions = null) => {
 	return promise;
 };
 
+reg.scheduleSync = async (delay = null, syncOptions = null) => {
+	reg.syncCalls_.push(true);
+	try {
+		await reg.scheduleSync_(delay, syncOptions);
+	} finally {
+		reg.syncCalls_.pop();
+	}
+};
+
 reg.setupRecurrentSync = () => {
 	if (reg.recurrentSyncId_) {
 		shim.clearInterval(reg.recurrentSyncId_);
@@ -183,4 +192,18 @@ reg.db = () => {
 	return reg.db_;
 };
 
+reg.cancelTimers = async () => {
+	if (this.recurrentSyncId_) clearTimeout(this.recurrentSyncId_);
+	return new Promise((resolve) => {
+		const iid = setInterval(() => {
+			if (!reg.syncCalls_.length) {
+				clearInterval(iid);
+				resolve();
+			}
+		}, 100);
+	});
+};
+
+reg.syncCalls_ = [];
+
 module.exports = { reg };

ReactNativeClient/lib/reserved-ids.js

@@ -0,0 +1,6 @@
+
+module.exports = Object.freeze({
+
+	ALL_NOTES_FILTER_ID: 'c3176726992c11e9ac940492261af972',
+
+});

ReactNativeClient/lib/services/DecryptionWorker.js

@@ -16,6 +16,8 @@ class DecryptionWorker {
 		this.eventEmitter_ = new EventEmitter();
 		this.kvStore_ = null;
 		this.maxDecryptionAttempts_ = 2;
+
+		this.startCalls_ = [];
 	}
 
 	setLogger(l) {
@@ -92,7 +94,7 @@ class DecryptionWorker {
 		this.dispatch(action);
 	}
 
-	async start(options = null) {
+	async start_(options = null) {
 		if (options === null) options = {};
 		if (!('masterKeyNotLoadedHandler' in options)) options.masterKeyNotLoadedHandler = 'throw';
 		if (!('errorHandler' in options)) options.errorHandler = 'log';
@@ -238,6 +240,27 @@ class DecryptionWorker {
 			this.scheduleStart();
 		}
 	}
+
+	async start(options) {
+		this.startCalls_.push(true);
+		try {
+			await this.start_(options);
+		} finally {
+			this.startCalls_.pop();
+		}
+	}
+
+	async cancelTimers() {
+		if (this.scheduleId_) clearTimeout(this.scheduleId_);
+		return new Promise((resolve) => {
+			const iid = setInterval(() => {
+				if (!this.startCalls_.length) {
+					clearInterval(iid);
+					resolve();
+				}
+			}, 100);
+		});
+	}
 }
 
 module.exports = DecryptionWorker;

ReactNativeClient/lib/services/SearchEngine.js

@@ -14,6 +14,7 @@ class SearchEngine {
 		this.logger_ = new Logger();
 		this.db_ = null;
 		this.isIndexing_ = false;
+		this.syncCalls_ = [];
 	}
 
 	static instance() {
@@ -95,7 +96,7 @@ class SearchEngine {
 		return this.syncTables();
 	}
 
-	async syncTables() {
+	async syncTables_() {
 		if (this.isIndexing_) return;
 
 		this.isIndexing_ = true;
@@ -176,6 +177,15 @@ class SearchEngine {
 		this.isIndexing_ = false;
 	}
 
+	async syncTables() {
+		this.syncCalls_.push(true);
+		try {
+			await this.syncTables_();
+		} finally {
+			this.syncCalls_.pop();
+		}
+	}
+
 	async countRows() {
 		const sql = 'SELECT count(*) as total FROM notes_fts';
 		const row = await this.db().selectOne(sql);
@@ -402,6 +412,18 @@ class SearchEngine {
 			}
 		}
 	}
+
+	async cancelTimers() {
+		if (this.scheduleSyncTablesIID_) clearTimeout(this.scheduleSyncTablesIID_);
+		return new Promise((resolve) => {
+			const iid = setInterval(() => {
+				if (!this.syncCalls_.length) {
+					clearInterval(iid);
+					resolve();
+				}
+			}, 100);
+		});
+	}
 }
 
 module.exports = SearchEngine;