Mobile: Fixes #382: Implemented new search engine for mobile and highlight searched words in notes

ReactNativeClient/lib/BaseApplication.js

@@ -34,7 +34,7 @@ const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js');
 const SyncTargetDropbox = require('lib/SyncTargetDropbox.js');
 const EncryptionService = require('lib/services/EncryptionService');
 const ResourceFetcher = require('lib/services/ResourceFetcher');
-const SearchEngine = require('lib/services/SearchEngine');
+const SearchEngineUtils = require('lib/services/SearchEngineUtils');
 const DecryptionWorker = require('lib/services/DecryptionWorker');
 const BaseService = require('lib/services/BaseService');
 
@@ -220,27 +220,7 @@ class BaseApplication {
 				notes = await Tag.notes(parentId, options);
 			} else if (parentType === BaseModel.TYPE_SEARCH) {
 				const search = BaseModel.byId(state.searches, parentId);
-				const results = await SearchEngine.instance().search(search.query_pattern);
-				const noteIds = results.map(n => n.id);
-
-				const previewOptions = {
-					order: [],
-					fields: Note.previewFields(),
-					conditions: ['id IN ("' + noteIds.join('","') + '")'],
-				}
-
-				notes = await Note.previews(null, previewOptions);
-
-				// By default, the notes will be returned in reverse order
-				// or maybe random order so sort them here in the correct order
-				// (search engine returns the results in order of relevance).
-				const sortedNotes = [];
-				for (let i = 0; i < notes.length; i++) {
-					const idx = noteIds.indexOf(notes[i].id);
-					sortedNotes[idx] = notes[i];
-				}
-
-				notes = sortedNotes;
+				notes = await SearchEngineUtils.notesForQuery(search.query_pattern);
 			}
 		}
 

ReactNativeClient/lib/MdToHtml.js

@@ -7,6 +7,7 @@ const { shim } = require('lib/shim.js');
 const { _ } = require('lib/locale');
 const md5 = require('md5');
 const MdToHtml_Katex = require('lib/MdToHtml_Katex');
+const { pregQuote } = require('lib/string-utils.js');
 
 class MdToHtml {
 
@@ -413,10 +414,30 @@ class MdToHtml {
 		return output.join('');
 	}
 
+	applyHighlightedKeywords_(body, keywords) {
+		for (let i = 0; i < keywords.length; i++) {
+			const k = keywords[i];
+
+			let regexString = '';
+
+			if (k.type === 'regex') {
+				regexString = k.value;
+			} else {
+				regexString = pregQuote(k);
+			}
+
+			const re = new RegExp('(^|\n|\b)(' + regexString + ')(\n|\b|$)', 'gi');
+			body = body.replace(re, '$1<span class="highlighted-keyword">$2</span>$3');
+		}
+
+		return body;
+	}
+
 	render(body, style, options = null) {
 		if (!options) options = {};
 		if (!options.postMessageSyntax) options.postMessageSyntax = 'postMessage';
 		if (!options.paddingBottom) options.paddingBottom = '0';
+		if (!options.highlightedKeywords) options.highlightedKeywords = [];
 
 		const cacheKey = this.makeContentKey(this.loadedResources_, body, style, options);
 		if (this.cachedContentKey_ === cacheKey) return this.cachedContent_;
@@ -427,38 +448,34 @@ class MdToHtml {
 			html: true,
 		});
 
+		body = this.applyHighlightedKeywords_(body, options.highlightedKeywords);
+
 		// Add `file:` protocol in linkify to allow text in the format of "file://..." to translate into
 		// file-URL links in html view
 		md.linkify.add('file:', {
-
-		  validate: function (text, pos, self) {
-		    var tail = text.slice(pos);
-
-		    if (!self.re.file) {
-		      self.re.file =  new RegExp(
-			'^[\\/]{2,3}[\\S]+' // matches all local file URI on Win/Unix/MacOS systems including reserved characters in some OS (i.e. no OS specific sanity check)
-		      );
-		    }
-		    if (self.re.file.test(tail)) {
-		      return tail.match(self.re.file)[0].length;
-		    }
-		    return 0;
-		}
-			
+		  	validate: function (text, pos, self) {
+			    var tail = text.slice(pos);
+			    if (!self.re.file) {
+			    	// matches all local file URI on Win/Unix/MacOS systems including reserved characters in some OS (i.e. no OS specific sanity check)
+					self.re.file =  new RegExp('^[\\/]{2,3}[\\S]+');
+			    }
+			    if (self.re.file.test(tail)) {
+					return tail.match(self.re.file)[0].length;
+			    }
+			    return 0;
+			}	
 		});
 
 		// enable file link URLs in MarkdownIt. Keeps other URL restrictions of MarkdownIt untouched.
 		// Format [link name](file://...)		
 		md.validateLink = function (url) {
-
 			var BAD_PROTO_RE = /^(vbscript|javascript|data):/;
 			var GOOD_DATA_RE = /^data:image\/(gif|png|jpeg|webp);/;
 
-	    		// url should be normalized at this point, and existing entities are decoded
+	    	// url should be normalized at this point, and existing entities are decoded
 			var str = url.trim().toLowerCase();
 
 			return BAD_PROTO_RE.test(str) ? (GOOD_DATA_RE.test(str) ? true : false) : true;
-
 		}
 
 		// This is currently used only so that the $expression$ and $$\nexpression\n$$ blocks are translated
@@ -609,6 +626,11 @@ class MdToHtml {
 				padding-left: .2em;
 			}
 
+			.highlighted-keyword {
+				background-color: #F3B717;
+				color: black;
+			}
+
 			/* 
 			This is to fix https://github.com/laurent22/joplin/issues/764
 			Without this, the tag attached to an equation float at an absoluate position of the page,

ReactNativeClient/lib/components/note-body-viewer.js

@@ -87,6 +87,7 @@ class NoteBodyViewer extends Component {
 				}, 100);
 			},
 			paddingBottom: '3.8em', // Extra bottom padding to make it possible to scroll past the action button (so that it doesn't overlap the text)
+			highlightedKeywords: this.props.highlightedKeywords,
 		};
 
 		let html = this.mdToHtml_.render(note ? note.body : '', this.props.webViewStyle, mdOptions);

ReactNativeClient/lib/components/screens/note.js

@@ -37,6 +37,7 @@ const AlarmService = require('lib/services/AlarmService.js');
 const { SelectDateTimeDialog } = require('lib/components/select-date-time-dialog.js');
 const ShareExtension = require('react-native-share-extension').default;
 const CameraView = require('lib/components/CameraView');
+const SearchEngine = require('lib/services/SearchEngine');
 
 import FileViewer from 'react-native-file-viewer';
 
@@ -587,12 +588,19 @@ class NoteScreenComponent extends BaseScreenComponent {
 				this.saveOneProperty('body', newBody);
 			};
 
+			let keywords = [];
+			if (this.props.searchQuery) {
+				const parsedQuery = SearchEngine.instance().parseQuery(this.props.searchQuery);
+				keywords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
+			}
+
 			bodyComponent = <NoteBodyViewer
 				onJoplinLinkClick={this.onJoplinLinkClick_}
 				ref="noteBodyViewer"
 				style={this.styles().noteBodyViewer}
 				webViewStyle={theme}
 				note={note}
+				highlightedKeywords={keywords}
 				onCheckboxChange={(newBody) => { onCheckboxChange(newBody) }}
 			/>
 		} else {
@@ -737,6 +745,7 @@ const NoteScreen = connect(
 			folderId: state.selectedFolderId,
 			itemType: state.selectedItemType,
 			folders: state.folders,
+			searchQuery: state.searchQuery,
 			theme: state.settings.theme,
 			sharedData: state.sharedData,
 			showAdvancedOptions: state.settings.showAdvancedOptions,

ReactNativeClient/lib/components/screens/search.js

@@ -9,6 +9,7 @@ const { NoteItem } = require('lib/components/note-item.js');
 const { BaseScreenComponent } = require('lib/components/base-screen.js');
 const { themeStyle } = require('lib/components/global-style.js');
 const { dialogs } = require('lib/dialogs.js');
+const SearchEngineUtils = require('lib/services/SearchEngineUtils');
 const DialogBox = require('react-native-dialogbox').default;
 
 class SearchScreenComponent extends BaseScreenComponent {
@@ -105,17 +106,22 @@ class SearchScreenComponent extends BaseScreenComponent {
 		let notes = []
 
 		if (query) {
-			let p = query.split(' ');
-			let temp = [];
-			for (let i = 0; i < p.length; i++) {
-				let t = p[i].trim();
-				if (!t) continue;
-				temp.push(t);
-			}
+			notes = await SearchEngineUtils.notesForQuery(query);
 
-			notes = await Note.previews(null, {
-				anywherePattern: '*' + temp.join('*') + '*',
-			});
+			// Keeping the code below in case of compatibility issue with old versions
+			// of Android and SQLite FTS.
+
+			// let p = query.split(' ');
+			// let temp = [];
+			// for (let i = 0; i < p.length; i++) {
+			// 	let t = p[i].trim();
+			// 	if (!t) continue;
+			// 	temp.push(t);
+			// }
+
+			// notes = await Note.previews(null, {
+			// 	anywherePattern: '*' + temp.join('*') + '*',
+			// });
 		}
 
 		if (!this.isMounted_) return;

ReactNativeClient/lib/services/SearchEngine.js

@@ -4,6 +4,7 @@ const ItemChange = require('lib/models/ItemChange.js');
 const Setting = require('lib/models/Setting.js');
 const Note = require('lib/models/Note.js');
 const BaseModel = require('lib/BaseModel.js');
+const { pregQuote } = require('lib/string-utils.js');
 
 class SearchEngine {
 
@@ -105,12 +106,9 @@ class SearchEngine {
 			term = term.substr(1);
 		}
 
-		const preg_quote = (str, delimiter) => {
-			return (str + '').replace(new RegExp('[.\\\\+*?\\[\\^\\]$(){}=!<>|:\\' + (delimiter || '') + '-]', 'g'), '\\$&');
-		} // [^ \t,\.,\+\-\\*?!={}<>\|:"\'\(\)[\]]
-		let regexString = preg_quote(term);
+		let regexString = pregQuote(term);
 		if (regexString[regexString.length - 1] === '*') {
-			regexString = regexString.substr(0, regexString.length - 2) + '[^' + preg_quote(' \t\n\r,.,+-*?!={}<>|:"\'()[]') + ']' + '*';
+			regexString = regexString.substr(0, regexString.length - 2) + '[^' + pregQuote(' \t\n\r,.,+-*?!={}<>|:"\'()[]') + ']' + '*';
 		}
 
 		return regexString;

ReactNativeClient/lib/services/SearchEngineUtils.js

@@ -0,0 +1,32 @@
+const SearchEngine = require('lib/services/SearchEngine');
+const Note = require('lib/models/Note');
+
+class SearchEngineUtils {
+
+	static async notesForQuery(query) {
+		const results = await SearchEngine.instance().search(query);
+		const noteIds = results.map(n => n.id);
+
+		const previewOptions = {
+			order: [],
+			fields: Note.previewFields(),
+			conditions: ['id IN ("' + noteIds.join('","') + '")'],
+		}
+
+		const notes = await Note.previews(null, previewOptions);
+
+		// By default, the notes will be returned in reverse order
+		// or maybe random order so sort them here in the correct order
+		// (search engine returns the results in order of relevance).
+		const sortedNotes = [];
+		for (let i = 0; i < notes.length; i++) {
+			const idx = noteIds.indexOf(notes[i].id);
+			sortedNotes[idx] = notes[i];
+		}
+
+		return sortedNotes;
+	}
+
+}
+
+module.exports = SearchEngineUtils;

ReactNativeClient/lib/string-utils.js

@@ -224,4 +224,8 @@ function escapeHtml(s) {
 		.replace(/'/g, "'");
 }
 
-module.exports = { removeDiacritics, escapeFilename, wrap, splitCommandString, padLeft, toTitleCase, escapeHtml };
+function pregQuote(str, delimiter) {
+	return (str + '').replace(new RegExp('[.\\\\+*?\\[\\^\\]$(){}=!<>|:\\' + (delimiter || '') + '-]', 'g'), '\\$&');
+}
+
+module.exports = { removeDiacritics, escapeFilename, wrap, splitCommandString, padLeft, toTitleCase, escapeHtml, pregQuote };

ReactNativeClient/root.js

@@ -52,6 +52,7 @@ const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
 const DropdownAlert = require('react-native-dropdownalert').default;
 const ShareExtension = require('react-native-share-extension').default;
 const ResourceFetcher = require('lib/services/ResourceFetcher');
+const SearchEngine = require('lib/services/SearchEngine');
 
 const SyncTargetRegistry = require('lib/SyncTargetRegistry.js');
 const SyncTargetOneDrive = require('lib/SyncTargetOneDrive.js');
@@ -495,6 +496,9 @@ async function initialize(dispatch) {
 	ResourceFetcher.instance().setLogger(reg.logger());
 	ResourceFetcher.instance().start();
 
+	SearchEngine.instance().setDb(reg.db());
+	SearchEngine.instance().setLogger(reg.logger());
+
 	reg.scheduleSync().then(() => {
 		// Wait for the first sync before updating the notifications, since synchronisation
 		// might change the notifications.