1
0
mirror of https://github.com/laurent22/joplin.git synced 2025-01-11 18:24:43 +02:00

Desktop: Fuzzy search (#3632)

This commit is contained in:
Naveen M V 2020-09-06 17:37:00 +05:30 committed by GitHub
parent a8296e2e37
commit e108fdb1d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 602 additions and 209 deletions

View File

@ -63,8 +63,6 @@ Modules/TinyMCE/langs/
# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
CliClient/app/LinkSelector.js
CliClient/build/LinkSelector.js
CliClient/tests-build/synchronizer_LockHandler.js
CliClient/tests-build/synchronizer_MigrationHandler.js
CliClient/tests/synchronizer_LockHandler.js
CliClient/tests/synchronizer_MigrationHandler.js
ElectronClient/commands/focusElement.js

4
.gitignore vendored
View File

@ -50,12 +50,12 @@ joplin-webclipper-source.zip
Tools/commit_hook.txt
.vscode/*
*.map
ReactNativeClient/lib/sql-extensions/
!ReactNativeClient/lib/sql-extensions/spellfix.dll
# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
CliClient/app/LinkSelector.js
CliClient/build/LinkSelector.js
CliClient/tests-build/synchronizer_LockHandler.js
CliClient/tests-build/synchronizer_MigrationHandler.js
CliClient/tests/synchronizer_LockHandler.js
CliClient/tests/synchronizer_MigrationHandler.js
ElectronClient/commands/focusElement.js

View File

@ -2,6 +2,9 @@ const gulp = require('gulp');
const fs = require('fs-extra');
const utils = require('../Tools/gulp/utils');
const tasks = {
compileExtensions: {
fn: require('../Tools/gulp/tasks/compileExtensions.js'),
},
copyLib: require('../Tools/gulp/tasks/copyLib'),
tsc: require('../Tools/gulp/tasks/tsc'),
updateIgnoredTypeScriptBuild: require('../Tools/gulp/tasks/updateIgnoredTypeScriptBuild'),
@ -53,10 +56,12 @@ utils.registerGulpTasks(gulp, tasks);
gulp.task('build', gulp.series([
'prepareBuild',
'compileExtensions',
'copyLib',
]));
gulp.task('buildTests', gulp.series([
'prepareTestBuild',
'compileExtensions',
'copyLib',
]));

View File

@ -4,7 +4,10 @@ require('app-module-path').addPath(__dirname);
const filterParser = require('lib/services/searchengine/filterParser.js').default;
// import filterParser from 'lib/services/searchengine/filterParser.js';
const makeTerm = (name, value, negated) => { return { name, value, negated }; };
const makeTerm = (name, value, negated, quoted = false) => {
if (name !== 'text') { return { name, value, negated }; } else { return { name, value, negated, quoted }; }
};
describe('filterParser should be correct filter for keyword', () => {
it('title', () => {
const searchString = 'title: something';
@ -65,7 +68,7 @@ describe('filterParser should be correct filter for keyword', () => {
it('phrase text search', () => {
const searchString = '"scott joplin"';
expect(filterParser(searchString)).toContain(makeTerm('text', '"scott joplin"', false));
expect(filterParser(searchString)).toContain(makeTerm('text', '"scott joplin"', false, true));
});
it('multi word body', () => {

View File

@ -426,7 +426,7 @@ describe('services_SearchEngine', function() {
const t = testCases[i];
const input = t[0];
const expected = t[1];
const actual = engine.parseQuery(input);
const actual = await engine.parseQuery(input);
const _Values = actual.terms._ ? actual.terms._.map(v => v.value) : undefined;
const titleValues = actual.terms.title ? actual.terms.title.map(v => v.value) : undefined;

View File

@ -0,0 +1,145 @@
/* eslint-disable no-unused-vars */
/* eslint prefer-const: 0*/
require('app-module-path').addPath(__dirname);
const { time } = require('lib/time-utils.js');
const { fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, asyncTest, db, synchronizer, fileApi, sleep, createNTestNotes, switchClient, createNTestFolders } = require('test-utils.js');
const SearchEngine = require('lib/services/searchengine/SearchEngine');
const Note = require('lib/models/Note');
const Folder = require('lib/models/Folder');
const Tag = require('lib/models/Tag');
const ItemChange = require('lib/models/ItemChange');
const Setting = require('lib/models/Setting');
const Resource = require('lib/models/Resource.js');
const { shim } = require('lib/shim');
const ResourceService = require('lib/services/ResourceService.js');
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
let engine = null;
const ids = (array) => array.map(a => a.id);
describe('services_SearchFuzzy', function() {
beforeEach(async (done) => {
await setupDatabaseAndSynchronizer(1);
await switchClient(1);
engine = new SearchEngine();
engine.setDb(db());
Setting.setValue('db.fuzzySearchEnabled', 1);
done();
});
it('should return note almost matching title', asyncTest(async () => {
let rows;
const n1 = await Note.save({ title: 'If It Ain\'t Baroque, Don\'t Fix It' });
const n2 = await Note.save({ title: 'Important note' });
await engine.syncTables();
rows = await engine.search('Broke', { fuzzy: false });
expect(rows.length).toBe(0);
rows = await engine.search('Broke', { fuzzy: true });
expect(rows.length).toBe(1);
expect(rows[0].id).toBe(n1.id);
rows = await engine.search('title:Broke', { fuzzy: true });
expect(rows.length).toBe(1);
expect(rows[0].id).toBe(n1.id);
rows = await engine.search('title:"Broke"', { fuzzy: true });
expect(rows.length).toBe(1);
expect(rows[0].id).toBe(n1.id);
rows = await engine.search('Imprtant', { fuzzy: true });
expect(rows.length).toBe(1);
expect(rows[0].id).toBe(n2.id);
}));
it('should order results by min fuzziness', asyncTest(async () => {
let rows;
const n1 = await Note.save({ title: 'I demand you take me to him' });
const n2 = await Note.save({ title: 'He demanded an answer' });
const n3 = await Note.save({ title: 'Don\'t you make demands of me' });
const n4 = await Note.save({ title: 'No drama for me' });
const n5 = await Note.save({ title: 'Just minding my own business' });
await engine.syncTables();
rows = await engine.search('demand', { fuzzy: false });
expect(rows.length).toBe(1);
expect(rows[0].id).toBe(n1.id);
rows = await engine.search('demand', { fuzzy: true });
expect(rows.length).toBe(3);
expect(rows[0].id).toBe(n1.id);
expect(rows[1].id).toBe(n3.id);
expect(rows[2].id).toBe(n2.id);
}));
it('should consider any:1', asyncTest(async () => {
let rows;
const n1 = await Note.save({ title: 'cat' });
const n2 = await Note.save({ title: 'cats' });
const n3 = await Note.save({ title: 'cot' });
const n4 = await Note.save({ title: 'defenestrate' });
const n5 = await Note.save({ title: 'defenstrate' });
const n6 = await Note.save({ title: 'defenestrated' });
const n7 = await Note.save({ title: 'he defenestrated the cat' });
await engine.syncTables();
rows = await engine.search('defenestrated cat', { fuzzy: true });
expect(rows.length).toBe(1);
rows = await engine.search('any:1 defenestrated cat', { fuzzy: true });
expect(rows.length).toBe(7);
}));
it('should leave phrase searches alone', asyncTest(async () => {
let rows;
const n1 = await Note.save({ title: 'abc def' });
const n2 = await Note.save({ title: 'def ghi' });
const n3 = await Note.save({ title: 'ghi jkl' });
const n4 = await Note.save({ title: 'def abc' });
const n5 = await Note.save({ title: 'mno pqr ghi jkl' });
await engine.syncTables();
rows = await engine.search('abc def', { fuzzy: true });
expect(rows.length).toBe(2);
expect(rows.map(r=>r.id)).toContain(n1.id);
expect(rows.map(r=>r.id)).toContain(n4.id);
rows = await engine.search('"abc def"', { fuzzy: true });
expect(rows.length).toBe(1);
expect(rows.map(r=>r.id)).toContain(n1.id);
rows = await engine.search('"ghi jkl"', { fuzzy: true });
expect(rows.length).toBe(2);
expect(rows.map(r=>r.id)).toContain(n3.id);
expect(rows.map(r=>r.id)).toContain(n5.id);
rows = await engine.search('"ghi jkl" mno', { fuzzy: true });
expect(rows.length).toBe(1);
expect(rows.map(r=>r.id)).toContain(n5.id);
rows = await engine.search('any:1 "ghi jkl" mno', { fuzzy: true });
expect(rows.length).toBe(2);
expect(rows.map(r=>r.id)).toContain(n3.id);
expect(rows.map(r=>r.id)).toContain(n5.id);
}));
});

View File

@ -4,6 +4,7 @@ const { themeStyle } = require('lib/theme');
const { _ } = require('lib/locale.js');
const { bridge } = require('electron').remote.require('./bridge');
const CommandService = require('lib/services/CommandService').default;
const Setting = require('lib/models/Setting.js');
const commands = [
require('./commands/focusSearch'),
@ -28,7 +29,7 @@ class HeaderComponent extends React.Component {
const triggerOnQuery = query => {
clearTimeout(this.scheduleSearchChangeEventIid_);
if (this.searchOnQuery_) this.searchOnQuery_(query);
if (this.searchOnQuery_) this.searchOnQuery_(query, Setting.value('db.fuzzySearchEnabled'));
this.scheduleSearchChangeEventIid_ = null;
};

View File

@ -445,8 +445,8 @@ class MainScreenComponent extends React.Component {
headerItems.push({
title: _('Search...'),
iconName: 'fa-search',
onQuery: query => {
CommandService.instance().execute('search', { query });
onQuery: (query, fuzzy = false) => {
CommandService.instance().execute('search', { query, fuzzy });
},
type: 'search',
});

View File

@ -10,7 +10,7 @@ export const declaration:CommandDeclaration = {
export const runtime = (comp:any):CommandRuntime => {
return {
execute: async ({ query }:any) => {
execute: async ({ query, fuzzy }:any) => {
console.info('RUNTIME', query);
if (!comp.searchId_) comp.searchId_ = uuid.create();
@ -23,6 +23,7 @@ export const runtime = (comp:any):CommandRuntime => {
query_pattern: query,
query_folder_id: null,
type_: BaseModel.TYPE_SEARCH,
fuzzy: fuzzy,
},
});

View File

@ -365,7 +365,7 @@ function NoteEditor(props: NoteEditorProps) {
);
}
const searchMarkers = useSearchMarkers(showLocalSearch, localSearchMarkerOptions, props.searches, props.selectedSearchId);
const searchMarkers = useSearchMarkers(showLocalSearch, localSearchMarkerOptions, props.searches, props.selectedSearchId, props.highlightedWords);
const editorProps:NoteBodyEditorProps = {
ref: editorRef,
@ -531,6 +531,7 @@ const mapStateToProps = (state: any) => {
customCss: state.customCss,
noteVisiblePanes: state.noteVisiblePanes,
watchedResources: state.watchedResources,
highlightedWords: state.highlightedWords,
};
};

View File

@ -23,6 +23,7 @@ export interface NoteEditorProps {
customCss: string,
noteVisiblePanes: string[],
watchedResources: any,
highlightedWords: any[],
}
export interface NoteBodyEditorProps {

View File

@ -1,8 +1,5 @@
import { useMemo } from 'react';
const BaseModel = require('lib/BaseModel.js');
const SearchEngine = require('lib/services/searchengine/SearchEngine');
interface SearchMarkersOptions {
searchTimestamp: number,
selectedIndex: number,
@ -25,18 +22,14 @@ function defaultSearchMarkers():SearchMarkers {
};
}
export default function useSearchMarkers(showLocalSearch:boolean, localSearchMarkerOptions:Function, searches:any[], selectedSearchId:string) {
export default function useSearchMarkers(showLocalSearch:boolean, localSearchMarkerOptions:Function, searches:any[], selectedSearchId:string, highlightedWords: any[] = []) {
return useMemo(():SearchMarkers => {
if (showLocalSearch) return localSearchMarkerOptions();
const output = defaultSearchMarkers();
const search = BaseModel.byId(searches, selectedSearchId);
if (search) {
const parsedQuery = SearchEngine.instance().parseQuery(search.query_pattern);
output.keywords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
}
output.keywords = highlightedWords;
return output;
}, [showLocalSearch, localSearchMarkerOptions, searches, selectedSearchId]);
}, [highlightedWords, showLocalSearch, localSearchMarkerOptions, searches, selectedSearchId]);
}

View File

@ -7,7 +7,6 @@ const BaseModel = require('lib/BaseModel');
const { _ } = require('lib/locale.js');
const { bridge } = require('electron').remote.require('./bridge');
const eventManager = require('lib/eventManager');
const SearchEngine = require('lib/services/searchengine/SearchEngine');
const Note = require('lib/models/Note');
const Setting = require('lib/models/Setting');
const NoteListUtils = require('../utils/NoteListUtils');
@ -229,8 +228,7 @@ class NoteListComponent extends React.Component {
if (this.props.notesParentType === 'Search') {
const query = BaseModel.byId(this.props.searches, this.props.selectedSearchId);
if (query) {
const parsedQuery = SearchEngine.instance().parseQuery(query.query_pattern);
return SearchEngine.instance().allParsedQueryTerms(parsedQuery);
return this.props.highlightedWords;
}
}
return [];
@ -460,6 +458,7 @@ const mapStateToProps = state => {
provisionalNoteIds: state.provisionalNoteIds,
isInsertingNotes: state.isInsertingNotes,
noteSortOrder: state.settings['notes.sortOrder.field'],
highlightedWords: state.highlightedWords,
};
};

View File

@ -17,6 +17,9 @@ const tasks = {
electronRebuild: {
fn: require('./tools/electronRebuild.js'),
},
compileExtensions: {
fn: require('../Tools/gulp/tasks/compileExtensions.js'),
},
copyLib: require('../Tools/gulp/tasks/copyLib'),
tsc: require('../Tools/gulp/tasks/tsc'),
updateIgnoredTypeScriptBuild: require('../Tools/gulp/tasks/updateIgnoredTypeScriptBuild'),
@ -25,6 +28,7 @@ const tasks = {
utils.registerGulpTasks(gulp, tasks);
const buildSeries = [
'compileExtensions',
'copyLib',
];

File diff suppressed because it is too large Load Diff

View File

@ -51,6 +51,10 @@
{
"from": "build-win/Joplin.VisualElementsManifest.xml",
"to": "."
},
{
"from": "lib/sql-extensions/spellfix.dll",
"to": "usr/lib/spellfix.dll"
}
],
"extraResources": [
@ -65,7 +69,13 @@
"artifactName": "${productName}Portable.${ext}"
},
"mac": {
"icon": "../../Assets/macOs.icns"
"icon": "../../Assets/macOs.icns",
"extraFiles": [
{
"from": "lib/sql-extensions/spellfix.dylib",
"to": "usr/lib/spellfix.dylib"
}
]
},
"linux": {
"icon": "../Assets/LinuxIcons/256x256.png",
@ -73,7 +83,13 @@
"desktop": {
"Icon": "joplin"
},
"target": "AppImage"
"target": "AppImage",
"extraFiles": [
{
"from": "lib/sql-extensions/spellfix.so",
"to": "usr/lib/spellfix.so"
}
]
}
},
"homepage": "https://github.com/laurent22/joplin#readme",

View File

@ -177,9 +177,8 @@ class Dialog extends React.PureComponent {
return output.join(' ');
}
keywords(searchQuery) {
const parsedQuery = SearchEngine.instance().parseQuery(searchQuery);
return SearchEngine.instance().allParsedQueryTerms(parsedQuery);
keywords() {
return this.props.highlightedWords;
}
markupToHtml() {
@ -227,7 +226,7 @@ class Dialog extends React.PureComponent {
}
} else {
const limit = 20;
const searchKeywords = this.keywords(searchQuery);
const searchKeywords = this.keywords();
const notes = await Note.byIds(results.map(result => result.id).slice(0, limit), { fields: ['id', 'body', 'markup_language', 'is_todo', 'todo_completed'] });
const notesById = notes.reduce((obj, { id, body, markup_language }) => ((obj[[id]] = { id, body, markup_language }), obj), {});
@ -283,7 +282,7 @@ class Dialog extends React.PureComponent {
this.setState({
listType: listType,
results: results,
keywords: this.keywords(searchQuery),
keywords: this.keywords(),
selectedItemId: results.length === 0 ? null : results[0].id,
resultsInBody: resultsInBody,
});
@ -455,6 +454,7 @@ const mapStateToProps = (state) => {
folders: state.folders,
theme: state.settings.theme,
showCompletedTodos: state.settings.showCompletedTodos,
highlightedWords: state.highlightedWords,
};
};

View File

@ -280,6 +280,7 @@ class BaseApplication {
});
let notes = [];
let highlightedWords = [];
if (parentId) {
if (parentType === Folder.modelType()) {
@ -288,12 +289,21 @@ class BaseApplication {
notes = await Tag.notes(parentId, options);
} else if (parentType === BaseModel.TYPE_SEARCH) {
const search = BaseModel.byId(state.searches, parentId);
notes = await SearchEngineUtils.notesForQuery(search.query_pattern);
notes = await SearchEngineUtils.notesForQuery(search.query_pattern, { fuzzy: search.fuzzy });
const parsedQuery = await SearchEngine.instance().parseQuery(search.query_pattern, search.fuzzy);
highlightedWords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
} else if (parentType === BaseModel.TYPE_SMART_FILTER) {
notes = await Note.previews(parentId, options);
}
}
if (highlightedWords.length) {
this.store().dispatch({
type: 'SET_HIGHLIGHTED',
words: highlightedWords,
});
}
this.store().dispatch({
type: 'NOTE_UPDATE_ALL',
notes: notes,
@ -681,6 +691,23 @@ class BaseApplication {
this.database_ = new JoplinDatabase(new DatabaseDriverNode());
this.database_.setLogExcludedQueryTypes(['SELECT']);
this.database_.setLogger(this.dbLogger_);
if (Setting.value('env') === 'dev') {
if (shim.isElectron()) {
this.database_.extensionToLoad = './lib/sql-extensions/spellfix';
}
} else {
if (shim.isElectron()) {
if (shim.isWindows()) {
const appDir = process.execPath.substring(0, process.execPath.lastIndexOf('\\'));
this.database_.extensionToLoad = `${appDir}/usr/lib/spellfix`;
} else {
const appDir = process.execPath.substring(0, process.execPath.lastIndexOf('/'));
this.database_.extensionToLoad = `${appDir}/usr/lib/spellfix`;
}
}
}
await this.database_.open({ name: `${profileDir}/database.sqlite` });
// if (Setting.value('env') === 'dev') await this.database_.clearForTesting();
@ -707,6 +734,11 @@ class BaseApplication {
setLocale(Setting.value('locale'));
}
if (Setting.value('db.fuzzySearchEnabled') === -1) {
const fuzzySearchEnabled = await this.database_.fuzzySearchEnabled();
Setting.setValue('db.fuzzySearchEnabled', fuzzySearchEnabled ? 1 : 0);
}
if (Setting.value('encryption.shouldReencrypt') < 0) {
// We suggest re-encryption if the user has at least one notebook
// and if encryption is enabled. This code runs only when shouldReencrypt = -1

View File

@ -40,7 +40,6 @@ const ImagePicker = require('react-native-image-picker');
const { SelectDateTimeDialog } = require('lib/components/select-date-time-dialog.js');
const ShareExtension = require('lib/ShareExtension.js').default;
const CameraView = require('lib/components/CameraView');
const SearchEngine = require('lib/services/searchengine/SearchEngine');
const urlUtils = require('lib/urlUtils');
class NoteScreenComponent extends BaseScreenComponent {
@ -975,8 +974,7 @@ class NoteScreenComponent extends BaseScreenComponent {
// Currently keyword highlighting is supported only when FTS is available.
let keywords = [];
if (this.props.searchQuery && !!this.props.ftsEnabled) {
const parsedQuery = SearchEngine.instance().parseQuery(this.props.searchQuery);
keywords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
keywords = this.props.highlightedWords;
}
// Note: as of 2018-12-29 it's important not to display the viewer if the note body is empty,
@ -1016,8 +1014,7 @@ class NoteScreenComponent extends BaseScreenComponent {
// Currently keyword highlighting is supported only when FTS is available.
let keywords = [];
if (this.props.searchQuery && !!this.props.ftsEnabled) {
const parsedQuery = SearchEngine.instance().parseQuery(this.props.searchQuery);
keywords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
keywords = this.props.highlightedWords;
}
const onCheckboxChange = newBody => {
@ -1202,6 +1199,7 @@ const NoteScreen = connect(state => {
sharedData: state.sharedData,
showSideMenu: state.showSideMenu,
provisionalNoteIds: state.provisionalNoteIds,
highlightedWords: state.highlightedWords,
};
})(NoteScreenComponent);

View File

@ -36,6 +36,10 @@ class DatabaseDriverNode {
});
}
loadExtension(path) {
return this.db_.loadExtension(path);
}
selectAll(sql, params = null) {
if (!params) params = {};
return new Promise((resolve, reject) => {

View File

@ -50,6 +50,10 @@ class DatabaseDriverReactNative {
});
}
loadExtension(path) {
throw new Error(`No extension support for ${path} in react-native-sqlite-storage`);
}
exec(sql, params = null) {
return new Promise((resolve, reject) => {
this.db_.executeSql(

View File

@ -97,6 +97,16 @@ class Database {
return this.tryCall('selectOne', sql, params);
}
async loadExtension(path) {
let result = null;
try {
result = await this.driver().loadExtension(path);
return result;
} catch (e) {
throw new Error(`Could not load extension ${path}`);
}
}
async selectAll(sql, params = null) {
return this.tryCall('selectAll', sql, params);
}

View File

@ -126,6 +126,7 @@ class JoplinDatabase extends Database {
this.tableFields_ = null;
this.version_ = null;
this.tableFieldNames_ = {};
this.extensionToLoad = './build/lib/sql-extensions/spellfix';
this.eventEmitter_ = new EventEmitter();
}
@ -283,6 +284,8 @@ class JoplinDatabase extends Database {
if (tableName == 'table_fields') continue;
if (tableName == 'sqlite_sequence') continue;
if (tableName.indexOf('notes_fts') === 0) continue;
if (tableName == 'notes_spellfix') continue;
if (tableName == 'search_aux') continue;
chain.push(() => {
return this.selectAll(`PRAGMA table_info("${tableName}")`).then(pragmas => {
for (let i = 0; i < pragmas.length; i++) {
@ -332,7 +335,7 @@ class JoplinDatabase extends Database {
// must be set in the synchronizer too.
// Note: v16 and v17 don't do anything. They were used to debug an issue.
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33];
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34];
let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
@ -848,13 +851,20 @@ class JoplinDatabase extends Database {
queries.push(this.addMigrationFile(33));
}
if (targetVersion == 34) {
queries.push('CREATE VIRTUAL TABLE search_aux USING fts4aux(notes_fts)');
queries.push('CREATE VIRTUAL TABLE notes_spellfix USING spellfix1');
}
queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });
try {
await this.transactionExecBatch(queries);
} catch (error) {
if (targetVersion === 15 || targetVersion === 18 || targetVersion === 33) {
this.logger().warn('Could not upgrade to database v15 or v18 or v33- FTS feature will not be used', error);
this.logger().warn('Could not upgrade to database v15 or v18 or v33 - FTS feature will not be used', error);
} else if (targetVersion === 34) {
this.logger().warn('Could not upgrade to database v34 - fuzzy search will not be used', error);
} else {
throw error;
}
@ -881,6 +891,17 @@ class JoplinDatabase extends Database {
return true;
}
async fuzzySearchEnabled() {
try {
await this.selectOne('SELECT count(*) FROM notes_spellfix');
} catch (error) {
this.logger().warn('Fuzzy search check failed', error);
return false;
}
this.logger().info('Fuzzy search check succeeded');
return true;
}
version() {
return this.version_;
}
@ -888,6 +909,12 @@ class JoplinDatabase extends Database {
async initialize() {
this.logger().info('Checking for database schema update...');
try {
await this.loadExtension(this.extensionToLoad);
} catch (error) {
console.info(error);
}
let versionRow = null;
try {
// Will throw if the database has not been created yet, but this is handled below

View File

@ -506,6 +506,7 @@ class Setting extends BaseModel {
'keychain.supported': { value: -1, type: Setting.TYPE_INT, public: false },
'db.ftsEnabled': { value: -1, type: Setting.TYPE_INT, public: false },
'db.fuzzySearchEnabled': { value: -1, type: Setting.TYPE_INT, public: false },
'encryption.enabled': { value: false, type: Setting.TYPE_BOOL, public: false },
'encryption.activeMasterKeyId': { value: '', type: Setting.TYPE_STRING, public: false },
'encryption.passwordCache': { value: {}, type: Setting.TYPE_OBJECT, public: false, secure: true },

View File

@ -12,6 +12,7 @@ const defaultState = {
masterKeys: [],
notLoadedMasterKeys: [],
searches: [],
highlightedWords: [],
selectedNoteIds: [],
selectedNoteHash: '',
selectedFolderId: null,
@ -946,6 +947,9 @@ const reducer = (state = defaultState, action) => {
}
newState.selectedNoteIds = [];
break;
case 'SET_HIGHLIGHTED':
newState = Object.assign({}, state, { highlightedWords: action.words });
break;
case 'APP_STATE_SET':
newState = Object.assign({}, state);

View File

@ -81,6 +81,11 @@ class SearchEngine {
);
}
if (!noteIds.length && (Setting.value('db.fuzzySearchEnabled') == 1)) {
// On the last loop
queries.push({ sql: 'INSERT INTO notes_spellfix(word,rank) SELECT term, documents FROM search_aux WHERE col=\'*\'' });
}
await this.db().transactionExecBatch(queries);
}
@ -145,7 +150,16 @@ class SearchEngine {
[BaseModel.TYPE_NOTE, lastChangeId]
);
if (!changes.length) break;
const queries = [];
if (!changes.length) {
if (Setting.value('db.fuzzySearchEnabled') === 1) {
queries.push({ sql: 'DELETE FROM notes_spellfix' });
queries.push({ sql: 'INSERT INTO notes_spellfix(word,rank) SELECT term, documents FROM search_aux WHERE col=\'*\'' });
await this.db().transactionExecBatch(queries);
}
break;
}
const noteIds = changes.map(a => a.item_id);
const notes = await Note.modelSelectAll(`
@ -153,8 +167,6 @@ class SearchEngine {
FROM notes WHERE id IN ("${noteIds.join('","')}") AND is_conflict = 0 AND encryption_applied = 0`
);
const queries = [];
for (let i = 0; i < changes.length; i++) {
const change = changes[i];
@ -254,7 +266,7 @@ class SearchEngine {
calculateWeightBM25_(rows) {
calculateWeightBM25_(rows, fuzzyScore) {
// https://www.sqlite.org/fts3.html#matchinfo
// pcnalx are the arguments passed to matchinfo
// p - The number of matchable phrases in the query.
@ -318,14 +330,20 @@ class SearchEngine {
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
row.weight = 0;
row.fuzziness = 1000;
row.wordFound = [];
for (let j = 0; j < numPhrases; j++) {
let found = false;
columns.forEach(column => {
const rowsWithHits = docsWithHits(X[i], column, j);
const frequencyHits = hitsThisRow(X[i], column, j);
const idf = IDF(rowsWithHits, numRows);
found = found ? found : (frequencyHits > 0);
row.weight += BM25(idf, frequencyHits, numTokens[column][i], avgTokens[column]);
row.fuzziness = (frequencyHits > 0) ? Math.min(row.fuzziness, fuzzyScore[j]) : row.fuzziness;
});
row.wordFound.push(found);
}
}
}
@ -345,23 +363,40 @@ class SearchEngine {
row.fields = Object.keys(matchedFields).filter(key => matchedFields[key]);
row.weight = 0;
row.fuzziness = 0;
}
}
processResults_(rows, parsedQuery, isBasicSearchResults = false) {
const rowContainsAllWords = (wordsFound, numFuzzyMatches) => {
let start = 0;
let end = 0;
for (let i = 0; i < numFuzzyMatches.length; i++) {
end = end + numFuzzyMatches[i];
if (!(wordsFound.slice(start, end).find(x => x))) {
// This note doesn't contain any fuzzy matches for the word
return false;
}
start = end;
}
return true;
};
if (isBasicSearchResults) {
this.processBasicSearchResults_(rows, parsedQuery);
} else {
this.calculateWeightBM25_(rows);
this.calculateWeightBM25_(rows, parsedQuery.fuzzyScore);
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
row.include = (parsedQuery.fuzzy && !parsedQuery.any) ? rowContainsAllWords(row.wordFound, parsedQuery.numFuzzyMatches) : true;
const offsets = row.offsets.split(' ').map(o => Number(o));
row.fields = this.fieldNamesFromOffsets_(offsets);
}
}
rows.sort((a, b) => {
if (a.fuzziness < b.fuzziness) return -1;
if (a.fuzziness > b.fuzziness) return +1;
if (a.fields.includes('title') && !b.fields.includes('title')) return -1;
if (!a.fields.includes('title') && b.fields.includes('title')) return +1;
if (a.weight < b.weight) return +1;
@ -389,21 +424,75 @@ class SearchEngine {
return regexString;
}
parseQuery(query) {
async fuzzifier(words) {
const fuzzyMatches = [];
words.forEach(word => {
const fuzzyWords = this.db().selectAll('SELECT word, score FROM notes_spellfix WHERE word MATCH ? AND top=3', [word]);
fuzzyMatches.push(fuzzyWords);
});
return await Promise.all(fuzzyMatches);
}
async parseQuery(query, fuzzy = false) {
// fuzzy = false;
const trimQuotes = (str) => str.startsWith('"') ? str.substr(1, str.length - 2) : str;
let allTerms = [];
let allFuzzyTerms = [];
try {
allTerms = filterParser(query);
} catch (error) {
console.warn(error);
}
const textTerms = allTerms.filter(x => x.name === 'text').map(x => trimQuotes(x.value));
const titleTerms = allTerms.filter(x => x.name === 'title').map(x => trimQuotes(x.value));
const bodyTerms = allTerms.filter(x => x.name === 'body').map(x => trimQuotes(x.value));
const textTerms = allTerms.filter(x => x.name === 'text' && !x.negated);
const titleTerms = allTerms.filter(x => x.name === 'title' && !x.negated);
const bodyTerms = allTerms.filter(x => x.name === 'body' && !x.negated);
const terms = { _: textTerms, 'title': titleTerms, 'body': bodyTerms };
const fuzzyScore = [];
let numFuzzyMatches = [];
let terms = null;
if (fuzzy) {
const fuzzyText = await this.fuzzifier(textTerms.filter(x => !x.quoted).map(x => trimQuotes(x.value)));
const fuzzyTitle = await this.fuzzifier(titleTerms.map(x => trimQuotes(x.value)));
const fuzzyBody = await this.fuzzifier(bodyTerms.map(x => trimQuotes(x.value)));
const phraseSearches = textTerms.filter(x => x.quoted).map(x => x.value);
// Save number of matches we got for each word
// fuzzifier() is currently set to return at most 3 matches)
// We need to know which fuzzy words go together so that we can filter out notes that don't contain a required word.
numFuzzyMatches = fuzzyText.concat(fuzzyTitle).concat(fuzzyBody).map(x => x.length);
for (let i = 0; i < phraseSearches.length; i++) {
numFuzzyMatches.push(1); // Phrase searches are preserved without fuzzification
}
const mergedFuzzyText = [].concat.apply([], fuzzyText);
const mergedFuzzyTitle = [].concat.apply([], fuzzyTitle);
const mergedFuzzyBody = [].concat.apply([], fuzzyBody);
const fuzzyTextTerms = mergedFuzzyText.map(x => { return { name: 'text', value: x.word, negated: false, score: x.score }; });
const fuzzyTitleTerms = mergedFuzzyTitle.map(x => { return { name: 'title', value: x.word, negated: false, score: x.score }; });
const fuzzyBodyTerms = mergedFuzzyBody.map(x => { return { name: 'body', value: x.word, negated: false, score: x.score }; });
const phraseTextTerms = phraseSearches.map(x => { return { name: 'text', value: x, negated: false, score: 0 }; });
allTerms = allTerms.filter(x => (x.name !== 'text' && x.name !== 'title' && x.name !== 'body'));
allFuzzyTerms = allTerms.concat(fuzzyTextTerms).concat(fuzzyTitleTerms).concat(fuzzyBodyTerms).concat(phraseTextTerms);
const allTextTerms = allFuzzyTerms.filter(x => x.name === 'title' || x.name === 'body' || x.name === 'text');
for (let i = 0; i < allTextTerms.length; i++) {
fuzzyScore.push(allFuzzyTerms[i].score ? allFuzzyTerms[i].score : 0);
}
terms = { _: fuzzyTextTerms.concat(phraseTextTerms).map(x =>trimQuotes(x.value)), 'title': fuzzyTitleTerms.map(x =>trimQuotes(x.value)), 'body': fuzzyBodyTerms.map(x =>trimQuotes(x.value)) };
} else {
const nonNegatedTextTerms = textTerms.length + titleTerms.length + bodyTerms.length;
for (let i = 0; i < nonNegatedTextTerms; i++) {
fuzzyScore.push(0);
}
terms = { _: textTerms.map(x =>trimQuotes(x.value)), 'title': titleTerms.map(x =>trimQuotes(x.value)), 'body': bodyTerms.map(x =>trimQuotes(x.value)) };
}
// Filter terms:
// - Convert wildcards to regex
@ -445,7 +534,11 @@ class SearchEngine {
termCount: termCount,
keys: keys,
terms: terms, // text terms
allTerms: allTerms,
allTerms: fuzzy ? allFuzzyTerms : allTerms,
fuzzyScore: fuzzyScore,
numFuzzyMatches: numFuzzyMatches,
fuzzy: fuzzy,
any: !!allTerms.find(term => term.name === 'any'),
};
}
@ -474,7 +567,7 @@ class SearchEngine {
async basicSearch(query) {
query = query.replace(/\*/, '');
const parsedQuery = this.parseQuery(query);
const parsedQuery = await this.parseQuery(query);
const searchOptions = {};
for (const key of parsedQuery.keys) {
@ -489,8 +582,8 @@ class SearchEngine {
return Note.previews(null, searchOptions);
}
determineSearchType_(query, preferredSearchType) {
if (preferredSearchType === SearchEngine.SEARCH_TYPE_BASIC) return SearchEngine.SEARCH_TYPE_BASIC;
determineSearchType_(query, options) {
if (options.searchType === SearchEngine.SEARCH_TYPE_BASIC) return SearchEngine.SEARCH_TYPE_BASIC;
// If preferredSearchType is "fts" we auto-detect anyway
// because it's not always supported.
@ -499,9 +592,12 @@ class SearchEngine {
if (!Setting.value('db.ftsEnabled') || ['ja', 'zh', 'ko', 'th'].indexOf(st) >= 0) {
return SearchEngine.SEARCH_TYPE_BASIC;
} else if ((Setting.value('db.fuzzySearchEnabled') === 1) && options.fuzzy) {
return SearchEngine.SEARCH_TYPE_FTS_FUZZY;
} else {
return SearchEngine.SEARCH_TYPE_FTS;
}
return SearchEngine.SEARCH_TYPE_FTS;
}
async search(searchString, options = null) {
@ -511,28 +607,31 @@ class SearchEngine {
searchString = this.normalizeText_(searchString);
const searchType = this.determineSearchType_(searchString, options.searchType);
const searchType = this.determineSearchType_(searchString, options);
if (searchType === SearchEngine.SEARCH_TYPE_BASIC) {
// Non-alphabetical languages aren't support by SQLite FTS (except with extensions which are not available in all platforms)
const rows = await this.basicSearch(searchString);
const parsedQuery = this.parseQuery(searchString);
const parsedQuery = await this.parseQuery(searchString);
this.processResults_(rows, parsedQuery, true);
return rows;
} else {
// SEARCH_TYPE_FTS
// SEARCH_TYPE_FTS or SEARCH_TYPE_FTS_FUZZY
// FTS will ignore all special characters, like "-" in the index. So if
// we search for "this-phrase" it won't find it because it will only
// see "this phrase" in the index. Because of this, we remove the dashes
// when searching.
// https://github.com/laurent22/joplin/issues/1075#issuecomment-459258856
const parsedQuery = this.parseQuery(searchString);
const parsedQuery = await this.parseQuery(searchString, options.fuzzy);
try {
const { query, params } = queryBuilder(parsedQuery.allTerms);
const { query, params } = (searchType === SearchEngine.SEARCH_TYPE_FTS_FUZZY) ? queryBuilder(parsedQuery.allTerms, true) : queryBuilder(parsedQuery.allTerms, false);
const rows = await this.db().selectAll(query, params);
this.processResults_(rows, parsedQuery);
if (searchType === SearchEngine.SEARCH_TYPE_FTS_FUZZY && !parsedQuery.any) {
return rows.filter(row => row.include);
}
return rows;
} catch (error) {
this.logger().warn(`Cannot execute MATCH query: ${searchString}: ${error.message}`);
@ -567,5 +666,6 @@ SearchEngine.instance_ = null;
SearchEngine.SEARCH_TYPE_AUTO = 'auto';
SearchEngine.SEARCH_TYPE_BASIC = 'basic';
SearchEngine.SEARCH_TYPE_FTS = 'fts';
SearchEngine.SEARCH_TYPE_FTS_FUZZY = 'fts_fuzzy';
module.exports = SearchEngine;

View File

@ -11,7 +11,7 @@ class SearchEngineUtils {
searchType = SearchEngine.SEARCH_TYPE_BASIC;
}
const results = await SearchEngine.instance().search(query, { searchType });
const results = await SearchEngine.instance().search(query, { searchType, fuzzy: options.fuzzy });
const noteIds = results.map(n => n.id);
// We need at least the note ID to be able to sort them below so if not

View File

@ -3,6 +3,7 @@ interface Term {
name: string
value: string
negated: boolean
quoted?: boolean
}
const makeTerm = (name: string, value: string): Term => {
@ -10,10 +11,9 @@ const makeTerm = (name: string, value: string): Term => {
return { name: name, value: value, negated: false };
};
const quoted = (s: string) => s.startsWith('"') && s.endsWith('"');
const quote = (s : string) => {
const quoted = (s: string) => s.startsWith('"') && s.endsWith('"');
if (!quoted(s)) {
return `"${s}"`;
}
@ -71,7 +71,6 @@ const parseQuery = (query: string): Term[] => {
'altitude', 'resource', 'sourceurl']);
const terms = getTerms(query);
// console.log(terms);
const result: Term[] = [];
for (let i = 0; i < terms.length; i++) {
@ -98,9 +97,9 @@ const parseQuery = (query: string): Term[] => {
// Every word is quoted if not already.
// By quoting the word, FTS match query will take care of removing dashes and other word seperators.
if (value.startsWith('-')) {
result.push({ name: 'text', value: quote(value.slice(1)) , negated: true });
result.push({ name: 'text', value: quote(value.slice(1)) , negated: true, quoted: quoted(value) });
} else {
result.push({ name: 'text', value: quote(value), negated: false });
result.push({ name: 'text', value: quote(value), negated: false, quoted: quoted(value) });
}
}
}

View File

@ -300,7 +300,7 @@ const sourceUrlFilter = (terms: Term[], conditons: string[], params: string[], r
};
const textFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation) => {
const textFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, fuzzy: Boolean) => {
const addExcludeTextConditions = (excludedTerms: Term[], conditions:string[], params: string[], relation: Relation) => {
const type = excludedTerms[0].name === 'text' ? '' : `.${excludedTerms[0].name}`;
@ -342,7 +342,7 @@ const textFilter = (terms: Term[], conditions: string[], params: string[], relat
if (term.name === 'text') return term.value;
else return `${term.name}:${term.value}`;
});
const matchQuery = (relation === 'OR') ? termsToMatch.join(' OR ') : termsToMatch.join(' ');
const matchQuery = (fuzzy || (relation === 'OR')) ? termsToMatch.join(' OR ') : termsToMatch.join(' ');
params.push(matchQuery);
}
@ -374,14 +374,11 @@ const getConnective = (terms: Term[], relation: Relation): string => {
return (!notebookTerm && (relation === 'OR')) ? 'ROWID=-1' : '1'; // ROWID=-1 acts as 0 (something always false)
};
export default function queryBuilder(terms: Term[]) {
export default function queryBuilder(terms: Term[], fuzzy: boolean) {
const queryParts: string[] = [];
const params: string[] = [];
const withs: string[] = [];
// console.log("testing beep beep boop boop")
// console.log(terms);
const relation: Relation = getDefaultRelation(terms);
queryParts.push(`
@ -405,8 +402,8 @@ export default function queryBuilder(terms: Term[]) {
resourceFilter(terms, queryParts, params, relation, withs);
textFilter(terms, queryParts, params, relation, fuzzy);
textFilter(terms, queryParts, params, relation);
typeFilter(terms, queryParts, params, relation);

Binary file not shown.

View File

@ -0,0 +1,50 @@
const fs = require('fs-extra');
const utils = require('../utils');
async function getSourceCode(dest) {
await utils.execCommand(`curl -o ${dest}/sqlite.tar.gz "https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release"`);
await utils.execCommand(`curl -o ${dest}/amalgamation.tar.gz "https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz"`);
await utils.execCommand(`tar -xzvf ${dest}/sqlite.tar.gz -C ${dest}`);
await utils.execCommand(`tar -xzvf ${dest}/amalgamation.tar.gz -C ${dest}`);
}
async function main() {
const rootDir = utils.rootDir();
const dest = `${rootDir}/ReactNativeClient/lib/sql-extensions`;
try {
await fs.ensureDir(dest);
if (utils.isLinux()) {
try {
await fs.promises.access(`${dest}/spellfix.so`);
} catch (e) {
await getSourceCode(dest);
await utils.execCommand(`gcc -shared -fPIC -Wall -I${dest}/sqlite-autoconf-3330000/ ${dest}/sqlite/ext/misc/spellfix.c -o ${dest}/spellfix.so`);
}
}
if (utils.isMac()) {
try {
await fs.promises.access(`${dest}/spellfix.dylib`);
} catch (e) {
await getSourceCode(dest);
await utils.execCommand(`gcc -shared -fPIC -Wall -I${dest}/sqlite-autoconf-3330000/ -dynamiclib ${dest}/sqlite/ext/misc/spellfix.c -o ${dest}/spellfix.dylib`);
}
}
// if (utils.isWindows()) {
// try {
// await fs.promises.access(`${dest}/spellfix.dll`);
// } catch (e) {
// await getSourceCode(dest);
// await utils.execCommand(`cl /I ${dest}/sqlite-autoconf-3330000 ${dest}/sqlite/ext/misc/spellfix.c -link -dll -out:spellfix.dll `)
// }
// }
} catch (e) {
console.warn(e);
}
}
module.exports = main;