From 96cd56548e58d537cff4757cc8ecd70effa255bc Mon Sep 17 00:00:00 2001
From: Laurent Cozic <laurent@cozic.net>
Date: Thu, 17 Jan 2019 19:01:35 +0000
Subject: [PATCH] Desktop: Improve search keyword highlighting

---
 .gitignore                                    |  1 +
 .../app/{compile-jsx.js => compile.js}        |  5 +-
 ElectronClient/app/gui/note-viewer/index.html | 15 +++--
 ElectronClient/app/package-lock.json          |  2 +-
 ElectronClient/app/package.json               |  4 +-
 ElectronClient/package-lock.json              | 11 ++++
 .../lib/services/SearchEngine.js              |  4 +-
 ReactNativeClient/lib/string-utils-common.js  | 55 ++++++++++++++++++
 ReactNativeClient/lib/string-utils.js         | 57 ++-----------------
 9 files changed, 90 insertions(+), 64 deletions(-)
 rename ElectronClient/app/{compile-jsx.js => compile.js} (86%)
 create mode 100644 ElectronClient/package-lock.json
 create mode 100644 ReactNativeClient/lib/string-utils-common.js

diff --git a/.gitignore b/.gitignore
index ff2de5f05..eef4769b5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,4 +40,5 @@ Tools/github_oauth_token.txt
 _releases
 ReactNativeClient/lib/csstojs/
 ElectronClient/app/gui/note-viewer/fonts/
+ElectronClient/app/gui/note-viewer/lib.js
 Tools/commit_hook.txt
\ No newline at end of file
diff --git a/ElectronClient/app/compile-jsx.js b/ElectronClient/app/compile.js
similarity index 86%
rename from ElectronClient/app/compile-jsx.js
rename to ElectronClient/app/compile.js
index 4663c7454..e605cc60e 100644
--- a/ElectronClient/app/compile-jsx.js
+++ b/ElectronClient/app/compile.js
@@ -2,6 +2,7 @@ const fs = require('fs-extra');
 const spawnSync	= require('child_process').spawnSync;
 
 const babelPath = __dirname + '/node_modules/.bin/babel' + (process.platform === 'win32' ? '.cmd' : '');
+const basePath = __dirname + '/../..';
 const guiPath = __dirname + '/gui';
 
 function fileIsNewerThan(path1, path2) {
@@ -27,7 +28,7 @@ fs.readdirSync(guiPath).forEach((filename) => {
 
 	if (fileIsNewerThan(jsxPath, jsPath)) {
 		console.info('Compiling ' + jsxPath + '...');
-		const result = spawnSync(babelPath, ['--presets', 'react', '--out-file',jsPath, jsxPath]);
+		const result = spawnSync(babelPath, ['--presets', 'react', '--out-file', jsPath, jsxPath]);
 		if (result.status !== 0) {
 			const msg = [];
 			if (result.stdout) msg.push(result.stdout.toString());
@@ -38,3 +39,5 @@ fs.readdirSync(guiPath).forEach((filename) => {
 		}
 	}
 });
+
+fs.copySync(basePath + '/ReactNativeClient/lib/string-utils-common.js', __dirname + '/gui/note-viewer/lib.js');
diff --git a/ElectronClient/app/gui/note-viewer/index.html b/ElectronClient/app/gui/note-viewer/index.html
index 49a84f1e4..9616a4d21 100644
--- a/ElectronClient/app/gui/note-viewer/index.html
+++ b/ElectronClient/app/gui/note-viewer/index.html
@@ -37,6 +37,7 @@
 	<div id="hlScriptContainer"></div>
 	<div id="markScriptContainer"></div>
 	<div id="content" ondragstart="return false;" ondrop="return false;"></div>
+	<script src="./lib.js"></script>
 
 	<script>
 	const contentElement = document.getElementById('content');
@@ -226,10 +227,6 @@
 	function setMarkers(keywords, options = null) {
 		if (!options) options = {};
 
-		// TODO: It should highlight queries without accents - eg "penche*" should highlight "penchés"
-		// TODO: It should highlight Chinese, Japanese characters, etc.
-		// TODO: It should highlight Russian
-		// TODO: not working - "oue*" doesn't highlight "ouéé"
 		// TODO: Search engine support to mobile app (sync tables)
 		// TODO: Update everywhere that uses allParsedQueryTerms to support new format
 		// TODO: Add support for scriptType on mobile and CLI
@@ -275,12 +272,20 @@
 			const isBasicSearch = ['ja', 'zh', 'ko'].indexOf(keyword.scriptType) >= 0;
 
 			if (keyword.type === 'regex') {
-				let regexString = '\\b' + keyword.value + '\\b';
+				const b = '[' + pregQuote(' \t\n\r,.,+-*?!={}<>|:"\'()[]') + ']+';
+
+				// The capturing groups are a hack to go around the strange behaviour of the ignoreGroups property. What we want is to
+				// exclude the first and last matches (the boundaries). What ignoreGroups does is ignore the first X groups. So
+				// we put the first boundary and the keyword inside a group, that way the first groups is ignored (ignoreGroups = 1)
+				// the second is included. And the last boundary is dropped because it's not in any group (it's important NOT to
+				// put this one in a group because otherwise it cannot be excluded).
+				let regexString = '(' + b + ')' + '(' + replaceRegexDiacritics(keyword.value) + ')' + b;
 				if (isBasicSearch) regexString = keyword.value;
 
 				mark_.markRegExp(new RegExp(regexString, 'gmi'), {
 					each: onEachElement,
 					acrossElements: true,
+					ignoreGroups: 1,
 				});
 			} else {
 				let accuracy = keyword.accuracy ? keyword.accuracy : 'exactly';
diff --git a/ElectronClient/app/package-lock.json b/ElectronClient/app/package-lock.json
index ee5a55842..298bb351b 100644
--- a/ElectronClient/app/package-lock.json
+++ b/ElectronClient/app/package-lock.json
@@ -4594,7 +4594,7 @@
     },
     "nan": {
       "version": "2.10.0",
-      "resolved": "http://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
       "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA=="
     },
     "nanomatch": {
diff --git a/ElectronClient/app/package.json b/ElectronClient/app/package.json
index 34ccac9f3..43f63b0f0 100644
--- a/ElectronClient/app/package.json
+++ b/ElectronClient/app/package.json
@@ -8,8 +8,8 @@
     "pack": "node_modules/.bin/electron-builder --dir",
     "dist": "node_modules/.bin/electron-builder",
     "publish": "build -p always",
-    "postinstall": "node compile-jsx.js && node compile-package-info.js && node ../../Tools/copycss.js --copy-fonts && install-app-deps",
-    "compile": "node compile-jsx.js && node compile-package-info.js && node ../../Tools/copycss.js --copy-fonts"
+    "postinstall": "node compile.js && node compile-package-info.js && node ../../Tools/copycss.js --copy-fonts && install-app-deps",
+    "compile": "node compile.js && node compile-package-info.js && node ../../Tools/copycss.js --copy-fonts"
   },
   "repository": {
     "type": "git",
diff --git a/ElectronClient/package-lock.json b/ElectronClient/package-lock.json
new file mode 100644
index 000000000..3ca31a954
--- /dev/null
+++ b/ElectronClient/package-lock.json
@@ -0,0 +1,11 @@
+{
+  "requires": true,
+  "lockfileVersion": 1,
+  "dependencies": {
+    "diacritics": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/diacritics/-/diacritics-1.3.0.tgz",
+      "integrity": "sha1-PvqHMj67hj5mls67AILUj/PW96E="
+    }
+  }
+}
diff --git a/ReactNativeClient/lib/services/SearchEngine.js b/ReactNativeClient/lib/services/SearchEngine.js
index a338d9a48..ce6c95aff 100644
--- a/ReactNativeClient/lib/services/SearchEngine.js
+++ b/ReactNativeClient/lib/services/SearchEngine.js
@@ -226,8 +226,8 @@ class SearchEngine {
 
 		let regexString = pregQuote(term);
 		if (regexString[regexString.length - 1] === '*') {
-			// regexString = regexString.substr(0, regexString.length - 2) + '[^' + pregQuote(' \t\n\r,.,+-*?!={}<>|:"\'()[]') + ']' + '*';
-			regexString = regexString.substr(0, regexString.length - 2) + '.*?';
+			regexString = regexString.substr(0, regexString.length - 2) + '[^' + pregQuote(' \t\n\r,.,+-*?!={}<>|:"\'()[]') + ']' + '*?';
+			// regexString = regexString.substr(0, regexString.length - 2) + '.*?';
 		}
 
 		return regexString;
diff --git a/ReactNativeClient/lib/string-utils-common.js b/ReactNativeClient/lib/string-utils-common.js
new file mode 100644
index 000000000..8e66c24f2
--- /dev/null
+++ b/ReactNativeClient/lib/string-utils-common.js
@@ -0,0 +1,55 @@
+function pregQuote(str, delimiter = '') {
+	return (str + '').replace(new RegExp('[.\\\\+*?\\[\\^\\]$(){}=!<>|:\\' + (delimiter || '') + '-]', 'g'), '\\$&');
+}
+
+function replaceRegexDiacritics(regexString) {
+	if (!regexString) return '';
+
+	const diacriticReplacements = {
+		'a': '[aàáâãäåāą]',
+		'A': '[AÀÁÂÃÄÅĀĄ]',
+		'c': '[cçćč]',
+		'C': '[CÇĆČ]',
+		'd': '[dđď]',
+		'D': '[DĐĎ]',
+		'e': '[eèéêëěēę]',
+		'E': '[EÈÉÊËĚĒĘ]',
+		'i': '[iìíîïī]',
+		'I': '[IÌÍÎÏĪ]',
+		'l': '[lł]',
+		'L': '[LŁ]',
+		'n': '[nñňń]',
+		'N': '[NÑŇŃ]',
+		'o': '[oòóôõöøō]',
+		'O': '[OÒÓÔÕÖØŌ]',
+		'r': '[rř]',
+		'R': '[RŘ]',
+		's': '[sšś]',
+		'S': '[SŠŚ]',
+		't': '[tť]',
+		'T': '[TŤ]',
+		'u': '[uùúûüůū]',
+		'U': '[UÙÚÛÜŮŪ]',
+		'y': '[yÿý]',
+		'Y': '[YŸÝ]',
+		'z': '[zžżź]',
+		'Z': '[ZŽŻŹ]',
+	};
+
+	let output = '';
+	for (let i = 0; i < regexString.length; i++) {
+		let c = regexString[i];
+		const r = diacriticReplacements[c];
+		if (r) {
+			output += r;
+		} else {
+			output += c;
+		}
+	}
+
+	return output;
+}
+
+if (typeof module !== 'undefined') {
+	module.exports = { pregQuote, replaceRegexDiacritics };
+}
diff --git a/ReactNativeClient/lib/string-utils.js b/ReactNativeClient/lib/string-utils.js
index 8e88d8ab3..26c9dd190 100644
--- a/ReactNativeClient/lib/string-utils.js
+++ b/ReactNativeClient/lib/string-utils.js
@@ -223,10 +223,6 @@ function escapeHtml(s) {
 		.replace(/'/g, "&#039;");
 }
 
-function pregQuote(str, delimiter = '') {
-	return (str + '').replace(new RegExp('[.\\\\+*?\\[\\^\\]$(){}=!<>|:\\' + (delimiter || '') + '-]', 'g'), '\\$&');
-}
-
 function surroundKeywords(keywords, text, prefix, suffix) {
 	if (!keywords.length) return text;
 
@@ -242,54 +238,6 @@ function surroundKeywords(keywords, text, prefix, suffix) {
 	return text.replace(re, prefix + '$1' + suffix);
 }
 
-function replaceRegexDiacritics(regexString) {
-	if (!regexString) return '';
-
-	const diacriticReplacements = {
-		'a': '[aàáâãäåāą]',
-		'A': '[AÀÁÂÃÄÅĀĄ]',
-		'c': '[cçćč]',
-		'C': '[CÇĆČ]',
-		'd': '[dđď]',
-		'D': '[DĐĎ]',
-		'e': '[eèéêëěēę]',
-		'E': '[EÈÉÊËĚĒĘ]',
-		'i': '[iìíîïī]',
-		'I': '[IÌÍÎÏĪ]',
-		'l': '[lł]',
-		'L': '[LŁ]',
-		'n': '[nñňń]',
-		'N': '[NÑŇŃ]',
-		'o': '[oòóôõöøō]',
-		'O': '[OÒÓÔÕÖØŌ]',
-		'r': '[rř]',
-		'R': '[RŘ]',
-		's': '[sšś]',
-		'S': '[SŠŚ]',
-		't': '[tť]',
-		'T': '[TŤ]',
-		'u': '[uùúûüůū]',
-		'U': '[UÙÚÛÜŮŪ]',
-		'y': '[yÿý]',
-		'Y': '[YŸÝ]',
-		'z': '[zžżź]',
-		'Z': '[ZŽŻŹ]',
-	};
-
-	let output = '';
-	for (let i = 0; i < regexString.length; i++) {
-		let c = regexString[i];
-		const r = diacriticReplacements[c];
-		if (r) {
-			output += r;
-		} else {
-			output += c;
-		}
-	}
-
-	return output;
-}
-
 const REGEX_JAPANESE = /[\u3000-\u303f]|[\u3040-\u309f]|[\u30a0-\u30ff]|[\uff00-\uff9f]|[\u4e00-\u9faf]|[\u3400-\u4dbf]/;
 const REGEX_CHINESE = /[\u4e00-\u9fff]|[\u3400-\u4dbf]|[\u{20000}-\u{2a6df}]|[\u{2a700}-\u{2b73f}]|[\u{2b740}-\u{2b81f}]|[\u{2b820}-\u{2ceaf}]|[\uf900-\ufaff]|[\u3300-\u33ff]|[\ufe30-\ufe4f]|[\uf900-\ufaff]|[\u{2f800}-\u{2fa1f}]/u;
 const REGEX_KOREAN = /[\uac00-\ud7af]|[\u1100-\u11ff]|[\u3130-\u318f]|[\ua960-\ua97f]|[\ud7b0-\ud7ff]/;
@@ -301,4 +249,7 @@ function scriptType(s) {
 	return 'en';
 }
 
-module.exports = { removeDiacritics, escapeFilename, wrap, splitCommandString, padLeft, toTitleCase, urlDecode, escapeHtml, pregQuote, surroundKeywords, scriptType, replaceRegexDiacritics };
+module.exports = Object.assign(
+	{ removeDiacritics, escapeFilename, wrap, splitCommandString, padLeft, toTitleCase, urlDecode, escapeHtml, surroundKeywords, scriptType },
+	require('./string-utils-common.js'),
+);