All: Added support for basic search

CliClient/tests/services_SearchEngine.js

@@ -344,7 +344,11 @@ describe('services_SearchEngine', function() {
 		let rows;
 
 		const testCases = [
-			['did-not-match', 'did-not-match'],
+			// "-" is considered a word delimiter so it is stripped off
+			// when indexing the notes. "did-not-match" is translated to
+			// three word "did", "not", "match"
+			['did-not-match', 'did not match'],
+			['did-not-match', '"did-not-match"'],
 			['does match', 'does match'],
 		];
 
@@ -358,8 +362,20 @@ describe('services_SearchEngine', function() {
 			rows = await engine.search(query);
 			expect(rows.length).toBe(1);
 
+
 			await Note.delete(n.id);
 		}
 	}));
 
+	it('should allow using basic search', asyncTest(async () => {
+		const n1 = await Note.save({ title: '- [ ] abcd' });
+		const n2 = await Note.save({ title: '[ ] abcd' });
+
+		await engine.syncTables();
+
+		expect((await engine.search('"- [ ]"', { searchType: SearchEngine.SEARCH_TYPE_FTS })).length).toBe(0);
+		expect((await engine.search('"- [ ]"', { searchType: SearchEngine.SEARCH_TYPE_BASIC })).length).toBe(1);
+		expect((await engine.search('"[ ]"', { searchType: SearchEngine.SEARCH_TYPE_BASIC })).length).toBe(2);
+	}));
+
 });

ElectronClient/gui/note-viewer/index.html

@@ -274,24 +274,31 @@
 			if (selectedElement && Date.now() - options.searchTimestamp <= 1000) selectedElement.scrollIntoView();
 		}
 
-		let markLoaded_ = false;
+		let markLoader_ = { state: 'idle', whenDone: null };
 		ipc.setMarkers = (event) => {
 			const keywords = event.keywords;
 			const options = event.options;
 
-			if (!keywords.length && !markLoaded_) return;
+			if (!keywords.length && markLoader_.state === 'idle') return;
+
+			if (markLoader_.state === 'idle') {
+				markLoader_ = {
+					state: 'loading',
+					whenDone: {keywords:keywords, options:options},
+				};
 
-			if (!markLoaded_) {
 				const script = document.createElement('script');
 				script.onload = function() {
-					setMarkers(keywords, options);
+					markLoader_.state = 'ready';
+					setMarkers(markLoader_.whenDone.keywords, markLoader_.whenDone.options);
 				};
 
 				script.src = '../../node_modules/mark.js/dist/mark.min.js';
 				document.getElementById('joplin-container-markScriptContainer').appendChild(script);
-				markLoaded_ = true;
-			} else {
+			} else if (markLoader_.state === 'ready') {
 				setMarkers(keywords, options);
+			} else if (markLoader_.state === 'loading') {
+				markLoader_.whenDone = {keywords:keywords, options:options};
 			}
 		}
 

README.md

@@ -296,6 +296,7 @@ Multiples words | Returns all the notes that contain **all** these words, but no
 Phrase query | Add double quotes to return the notes that contain exactly this phrase. | `"shopping list"` - will return the notes that contain these **exact terms** next to each other and in this order. It will **not** return for example a note that contains "going shopping with my list".
 Prefix | Add a wildcard to return all the notes that contain a term with a specified prefix. | `swim*` - will return all the notes that contain eg. "swim", but also "swimming", "swimsuit", etc. IMPORTANT: The wildcard **can only be at the end** - it will be ignored at the beginning of a word (eg. `*swim`) and will be treated as a literal asterisk in the middle of a word (eg. `ast*rix`)
 Field restricted | Add either `title:` or `body:` before a note to restrict your search to just the title, or just the body. | `title:shopping`, `body:egg`
+Switch to basic search | One drawback of Full Text Search is that it ignores most non-alphabetical characters. However in some cases you might want to search for this too. To do that, you can use basic search. You switch to this mode by prefixing your search with a slash `/`. This won't provide the benefits of FTS but it will allow searching exactly for what you need. Note that it can also be much slower, even extremely slow, depending on your query. | `/"- [ ]"` - will return all the notes that contain unchecked checkboxes.
 
 Notes are sorted by "relevance". Currently it means the notes that contain the requested terms the most times are on top. For queries with multiple terms, it also matters how close to each other the terms are. This is a bit experimental so if you notice a search query that returns unexpected results, please report it in the forum, providing as many details as possible to replicate the issue.
 

ReactNativeClient/lib/services/SearchEngine.js

@@ -392,18 +392,55 @@ class SearchEngine {
 		return Note.previews(null, searchOptions);
 	}
 
-	async search(query) {
-		query = this.normalizeText_(query);
-		query = query.replace(/-/g, ' '); // https://github.com/laurent22/joplin/issues/1075#issuecomment-459258856
+	determineSearchType_(query, preferredSearchType) {
+		if (preferredSearchType === SearchEngine.SEARCH_TYPE_BASIC) return SearchEngine.SEARCH_TYPE_BASIC;
+
+		// If preferredSearchType is "fts" we auto-detect anyway
+		// because it's not always supported.
 
 		const st = scriptType(query);
 
 		if (!Setting.value('db.ftsEnabled') || ['ja', 'zh', 'ko', 'th'].indexOf(st) >= 0) {
+			return SearchEngine.SEARCH_TYPE_BASIC;
+		}
+
+		return SearchEngine.SEARCH_TYPE_FTS;
+	}
+
+	async search(query, options = null) {
+		options = Object.assign({}, {
+			searchType: SearchEngine.SEARCH_TYPE_AUTO,
+		}, options);
+
+		query = this.normalizeText_(query);
+
+		const searchType = this.determineSearchType_(query, options.searchType);
+
+		if (searchType === SearchEngine.SEARCH_TYPE_BASIC) {
 			// Non-alphabetical languages aren't support by SQLite FTS (except with extensions which are not available in all platforms)
 			return this.basicSearch(query);
-		} else {
+		} else { // SEARCH_TYPE_FTS
+			// FTS will ignore all special characters, like "-" in the index. So if
+			// we search for "this-phrase" it won't find it because it will only
+			// see "this phrase" in the index. Because of this, we remove the dashes
+			// when searching.
+			// https://github.com/laurent22/joplin/issues/1075#issuecomment-459258856
+			query = query.replace(/-/g, ' ');
 			const parsedQuery = this.parseQuery(query);
-			const sql = 'SELECT notes_fts.id, notes_fts.title AS normalized_title, offsets(notes_fts) AS offsets, notes.title, notes.user_updated_time, notes.is_todo, notes.todo_completed, notes.parent_id FROM notes_fts LEFT JOIN notes ON notes_fts.id = notes.id WHERE notes_fts MATCH ?';
+			const sql = `
+				SELECT
+					notes_fts.id,
+					notes_fts.title AS normalized_title,
+					offsets(notes_fts) AS offsets,
+					notes.title,
+					notes.user_updated_time,
+					notes.is_todo,
+					notes.todo_completed,
+					notes.parent_id
+				FROM notes_fts
+				LEFT JOIN notes ON notes_fts.id = notes.id
+				WHERE notes_fts MATCH ?
+			`;
 			try {
 				const rows = await this.db().selectAll(sql, [query]);
 				this.processResults_(rows, parsedQuery);
@@ -436,4 +473,8 @@ class SearchEngine {
 
 SearchEngine.instance_ = null;
 
+SearchEngine.SEARCH_TYPE_AUTO = 'auto';
+SearchEngine.SEARCH_TYPE_BASIC = 'basic';
+SearchEngine.SEARCH_TYPE_FTS = 'fts';
+
 module.exports = SearchEngine;

ReactNativeClient/lib/services/SearchEngineUtils.js

@@ -5,7 +5,13 @@ class SearchEngineUtils {
 	static async notesForQuery(query, options = null) {
 		if (!options) options = {};
 
-		const results = await SearchEngine.instance().search(query);
+		let searchType = SearchEngine.SEARCH_TYPE_FTS;
+		if (query.length && query[0] === '/') {
+			query = query.substr(1);
+			searchType = SearchEngine.SEARCH_TYPE_BASIC;
+		}
+
+		const results = await SearchEngine.instance().search(query, { searchType });
 		const noteIds = results.map(n => n.id);
 
 		// We need at least the note ID to be able to sort them below so if not
@@ -18,15 +24,11 @@ class SearchEngineUtils {
 			idWasAutoAdded = true;
 		}
 
-		const previewOptions = Object.assign(
-			{},
-			{
-				order: [],
-				fields: fields,
-				conditions: [`id IN ("${noteIds.join('","')}")`],
-			},
-			options
-		);
+		const previewOptions = Object.assign({}, {
+			order: [],
+			fields: fields,
+			conditions: [`id IN ("${noteIds.join('","')}")`],
+		}, options);
 
 		const notes = await Note.previews(null, previewOptions);