Started integrating search engine to desktop app

ElectronClient/app/app.js

@@ -23,6 +23,7 @@ const DecryptionWorker = require('lib/services/DecryptionWorker');
 const InteropService = require('lib/services/InteropService');
 const InteropServiceHelper = require('./InteropServiceHelper.js');
 const ResourceService = require('lib/services/ResourceService');
+const SearchEngine = require('lib/services/SearchEngine');
 const ClipperServer = require('lib/ClipperServer');
 const ExternalEditWatcher = require('lib/services/ExternalEditWatcher');
 const { bridge } = require('electron').remote.require('./bridge');
@@ -793,6 +794,10 @@ class Application extends BaseApplication {
 
 		ResourceService.runInBackground();
 
+		SearchEngine.instance().setDb(reg.db());
+		SearchEngine.instance().setLogger(reg.logger());
+		SearchEngine.runInBackground();
+
 		if (Setting.value('env') === 'dev') {
 			AlarmService.updateAllNotifications();
 		} else {

ReactNativeClient/lib/BaseApplication.js

@@ -34,6 +34,7 @@ const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js');
 const SyncTargetDropbox = require('lib/SyncTargetDropbox.js');
 const EncryptionService = require('lib/services/EncryptionService');
 const ResourceFetcher = require('lib/services/ResourceFetcher');
+const SearchEngine = require('lib/services/SearchEngine');
 const DecryptionWorker = require('lib/services/DecryptionWorker');
 const BaseService = require('lib/services/BaseService');
 
@@ -218,12 +219,28 @@ class BaseApplication {
 			} else if (parentType === Tag.modelType()) {
 				notes = await Tag.notes(parentId, options);
 			} else if (parentType === BaseModel.TYPE_SEARCH) {
-				let fields = Note.previewFields();
-				let search = BaseModel.byId(state.searches, parentId);
-				notes = await Note.previews(null, {
-					fields: fields,
-					anywherePattern: '*' + search.query_pattern + '*',
-				});
+				const search = BaseModel.byId(state.searches, parentId);
+				const results = await SearchEngine.instance().search(search.query_pattern);
+				const noteIds = results.map(n => n.id);
+
+				const previewOptions = {
+					order: [],
+					fields: Note.previewFields(),
+					conditions: ['id IN ("' + noteIds.join('","') + '")'],
+				}
+
+				notes = await Note.previews(null, previewOptions);
+
+				// By default, the notes will be returned in reverse order
+				// or maybe random order so sort them here in the correct order
+				// (search engine returns the results in order of relevance).
+				const sortedNotes = [];
+				for (let i = 0; i < notes.length; i++) {
+					const idx = noteIds.indexOf(notes[i].id);
+					sortedNotes[idx] = notes[i];
+				}
+
+				notes = sortedNotes;
 			}
 		}
 

ReactNativeClient/lib/joplin-database.js

@@ -219,6 +219,7 @@ class JoplinDatabase extends Database {
 				if (tableName == 'android_metadata') continue;
 				if (tableName == 'table_fields') continue;
 				if (tableName == 'sqlite_sequence') continue;
+				if (tableName.indexOf('notes_fts') === 0) continue;
 				chain.push(() => {
 					return this.selectAll('PRAGMA table_info("' + tableName + '")').then((pragmas) => {
 						for (let i = 0; i < pragmas.length; i++) {
@@ -260,7 +261,7 @@ class JoplinDatabase extends Database {
 		// default value and thus might cause problems. In that case, the default value
 		// must be set in the synchronizer too.
 
-		const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14];
+		const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
 
 		let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
 
@@ -445,6 +446,12 @@ class JoplinDatabase extends Database {
 				}));
 			}
 
+			if (targetVersion == 15) {
+				// NOTE: Duplicated from SearchEngine.createFtsTables()
+				queries.push('CREATE VIRTUAL TABLE notes_fts USING fts4(content="notes", notindexed="id", id, title, body)');
+				queries.push('INSERT INTO notes_fts(docid, id, title, body) SELECT rowid, id, title, body FROM notes WHERE is_conflict = 0 AND encryption_applied = 0');
+			}
+
 			queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });
 			await this.transactionExecBatch(queries);
 

ReactNativeClient/lib/models/Note.js

@@ -249,7 +249,7 @@ class Note extends BaseItem {
 		// is used to sort already loaded notes.
 
 		if (!options) options = {};
-		if (!options.order) options.order = [
+		if (!('order' in options)) options.order = [
 			{ by: 'user_updated_time', dir: 'DESC' },
 			{ by: 'user_created_time', dir: 'DESC' },
 			{ by: 'title', dir: 'DESC' },

ReactNativeClient/lib/services/SearchEngine.js

@@ -1,4 +1,5 @@
 const { Logger } = require('lib/logger.js');
+const { shim } = require('lib/shim.js');
 const ItemChange = require('lib/models/ItemChange.js');
 const Setting = require('lib/models/Setting.js');
 const Note = require('lib/models/Note.js');
@@ -12,6 +13,7 @@ class SearchEngine {
 		this.db_ = null;
 	}
 
+	// Note: Duplicated in JoplinDatabase migration 15
 	async createFtsTables() {
 		await this.db().exec('CREATE VIRTUAL TABLE notes_fts USING fts4(content="notes", notindexed="id", id, title, body)');
 		await this.db().exec('INSERT INTO notes_fts(docid, id, title, body) SELECT rowid, id, title, body FROM notes WHERE is_conflict = 0 AND encryption_applied = 0');
@@ -27,6 +29,8 @@ class SearchEngine {
 		// await this.db().exec('INSERT INTO notes_fts(docid, id, title, body) SELECT rowid, id, title, body FROM notes WHERE is_conflict = 0 AND encryption_applied = 0');
 		// return;
 
+		this.logger().info('SearchEngine: Updating FTS table...');
+
 		await ItemChange.waitForAllSaved();
 
 		const startTime = Date.now();
@@ -113,6 +117,10 @@ class SearchEngine {
 	calculateWeight_(offsets) {
 		// Offset doc: https://www.sqlite.org/fts3.html#offsets
 
+		// TODO: If there's only one term - specia case - whatever has the most occurences win.
+		// TODO: Parse query string.
+		// TODO: Support wildcards
+		
 		const occurenceCount = Math.floor(offsets.length / 4);
 
 		let spread = 0;
@@ -151,12 +159,26 @@ class SearchEngine {
 	}
 
 	async search(query) {
-		const sql = 'SELECT id, offsets(notes_fts) AS offsets FROM notes_fts WHERE notes_fts MATCH ?'
+		const sql = 'SELECT id, title, offsets(notes_fts) AS offsets FROM notes_fts WHERE notes_fts MATCH ?'
 		const rows = await this.db().selectAll(sql, [query]);
 		this.orderResults_(rows);
 		return rows;
 	}
 
+	static runInBackground() {
+		if (this.isRunningInBackground_) return;
+
+		this.isRunningInBackground_ = true;
+
+		setTimeout(() => {
+			SearchEngine.instance().updateFtsTables();
+		}, 1000 * 30);
+		
+		shim.setInterval(() => {
+			SearchEngine.instance().updateFtsTables();
+		}, 1000 * 60 * 30);
+	}
+
 }
 
 module.exports = SearchEngine;