mirror of
https://github.com/laurent22/joplin.git
synced 2025-01-11 18:24:43 +02:00
Desktop: Fuzzy search (#3632)
This commit is contained in:
parent
a8296e2e37
commit
e108fdb1d8
@ -63,8 +63,6 @@ Modules/TinyMCE/langs/
|
||||
# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
|
||||
CliClient/app/LinkSelector.js
|
||||
CliClient/build/LinkSelector.js
|
||||
CliClient/tests-build/synchronizer_LockHandler.js
|
||||
CliClient/tests-build/synchronizer_MigrationHandler.js
|
||||
CliClient/tests/synchronizer_LockHandler.js
|
||||
CliClient/tests/synchronizer_MigrationHandler.js
|
||||
ElectronClient/commands/focusElement.js
|
||||
|
4
.gitignore
vendored
4
.gitignore
vendored
@ -50,12 +50,12 @@ joplin-webclipper-source.zip
|
||||
Tools/commit_hook.txt
|
||||
.vscode/*
|
||||
*.map
|
||||
ReactNativeClient/lib/sql-extensions/
|
||||
!ReactNativeClient/lib/sql-extensions/spellfix.dll
|
||||
|
||||
# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
|
||||
CliClient/app/LinkSelector.js
|
||||
CliClient/build/LinkSelector.js
|
||||
CliClient/tests-build/synchronizer_LockHandler.js
|
||||
CliClient/tests-build/synchronizer_MigrationHandler.js
|
||||
CliClient/tests/synchronizer_LockHandler.js
|
||||
CliClient/tests/synchronizer_MigrationHandler.js
|
||||
ElectronClient/commands/focusElement.js
|
||||
|
@ -2,6 +2,9 @@ const gulp = require('gulp');
|
||||
const fs = require('fs-extra');
|
||||
const utils = require('../Tools/gulp/utils');
|
||||
const tasks = {
|
||||
compileExtensions: {
|
||||
fn: require('../Tools/gulp/tasks/compileExtensions.js'),
|
||||
},
|
||||
copyLib: require('../Tools/gulp/tasks/copyLib'),
|
||||
tsc: require('../Tools/gulp/tasks/tsc'),
|
||||
updateIgnoredTypeScriptBuild: require('../Tools/gulp/tasks/updateIgnoredTypeScriptBuild'),
|
||||
@ -53,10 +56,12 @@ utils.registerGulpTasks(gulp, tasks);
|
||||
|
||||
gulp.task('build', gulp.series([
|
||||
'prepareBuild',
|
||||
'compileExtensions',
|
||||
'copyLib',
|
||||
]));
|
||||
|
||||
gulp.task('buildTests', gulp.series([
|
||||
'prepareTestBuild',
|
||||
'compileExtensions',
|
||||
'copyLib',
|
||||
]));
|
||||
|
@ -4,7 +4,10 @@ require('app-module-path').addPath(__dirname);
|
||||
const filterParser = require('lib/services/searchengine/filterParser.js').default;
|
||||
// import filterParser from 'lib/services/searchengine/filterParser.js';
|
||||
|
||||
const makeTerm = (name, value, negated) => { return { name, value, negated }; };
|
||||
const makeTerm = (name, value, negated, quoted = false) => {
|
||||
if (name !== 'text') { return { name, value, negated }; } else { return { name, value, negated, quoted }; }
|
||||
};
|
||||
|
||||
describe('filterParser should be correct filter for keyword', () => {
|
||||
it('title', () => {
|
||||
const searchString = 'title: something';
|
||||
@ -65,7 +68,7 @@ describe('filterParser should be correct filter for keyword', () => {
|
||||
|
||||
it('phrase text search', () => {
|
||||
const searchString = '"scott joplin"';
|
||||
expect(filterParser(searchString)).toContain(makeTerm('text', '"scott joplin"', false));
|
||||
expect(filterParser(searchString)).toContain(makeTerm('text', '"scott joplin"', false, true));
|
||||
});
|
||||
|
||||
it('multi word body', () => {
|
||||
|
@ -426,7 +426,7 @@ describe('services_SearchEngine', function() {
|
||||
const t = testCases[i];
|
||||
const input = t[0];
|
||||
const expected = t[1];
|
||||
const actual = engine.parseQuery(input);
|
||||
const actual = await engine.parseQuery(input);
|
||||
|
||||
const _Values = actual.terms._ ? actual.terms._.map(v => v.value) : undefined;
|
||||
const titleValues = actual.terms.title ? actual.terms.title.map(v => v.value) : undefined;
|
||||
|
145
CliClient/tests/services_SearchFuzzy.js
Normal file
145
CliClient/tests/services_SearchFuzzy.js
Normal file
@ -0,0 +1,145 @@
|
||||
/* eslint-disable no-unused-vars */
|
||||
/* eslint prefer-const: 0*/
|
||||
|
||||
require('app-module-path').addPath(__dirname);
|
||||
|
||||
const { time } = require('lib/time-utils.js');
|
||||
const { fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, asyncTest, db, synchronizer, fileApi, sleep, createNTestNotes, switchClient, createNTestFolders } = require('test-utils.js');
|
||||
const SearchEngine = require('lib/services/searchengine/SearchEngine');
|
||||
const Note = require('lib/models/Note');
|
||||
const Folder = require('lib/models/Folder');
|
||||
const Tag = require('lib/models/Tag');
|
||||
const ItemChange = require('lib/models/ItemChange');
|
||||
const Setting = require('lib/models/Setting');
|
||||
const Resource = require('lib/models/Resource.js');
|
||||
const { shim } = require('lib/shim');
|
||||
const ResourceService = require('lib/services/ResourceService.js');
|
||||
|
||||
|
||||
process.on('unhandledRejection', (reason, p) => {
|
||||
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
|
||||
});
|
||||
|
||||
let engine = null;
|
||||
|
||||
const ids = (array) => array.map(a => a.id);
|
||||
|
||||
describe('services_SearchFuzzy', function() {
|
||||
beforeEach(async (done) => {
|
||||
await setupDatabaseAndSynchronizer(1);
|
||||
await switchClient(1);
|
||||
|
||||
engine = new SearchEngine();
|
||||
engine.setDb(db());
|
||||
|
||||
Setting.setValue('db.fuzzySearchEnabled', 1);
|
||||
done();
|
||||
});
|
||||
|
||||
|
||||
it('should return note almost matching title', asyncTest(async () => {
|
||||
let rows;
|
||||
const n1 = await Note.save({ title: 'If It Ain\'t Baroque, Don\'t Fix It' });
|
||||
const n2 = await Note.save({ title: 'Important note' });
|
||||
|
||||
await engine.syncTables();
|
||||
rows = await engine.search('Broke', { fuzzy: false });
|
||||
expect(rows.length).toBe(0);
|
||||
|
||||
rows = await engine.search('Broke', { fuzzy: true });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].id).toBe(n1.id);
|
||||
|
||||
|
||||
rows = await engine.search('title:Broke', { fuzzy: true });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].id).toBe(n1.id);
|
||||
|
||||
rows = await engine.search('title:"Broke"', { fuzzy: true });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].id).toBe(n1.id);
|
||||
|
||||
rows = await engine.search('Imprtant', { fuzzy: true });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].id).toBe(n2.id);
|
||||
}));
|
||||
|
||||
|
||||
it('should order results by min fuzziness', asyncTest(async () => {
|
||||
let rows;
|
||||
const n1 = await Note.save({ title: 'I demand you take me to him' });
|
||||
const n2 = await Note.save({ title: 'He demanded an answer' });
|
||||
const n3 = await Note.save({ title: 'Don\'t you make demands of me' });
|
||||
const n4 = await Note.save({ title: 'No drama for me' });
|
||||
const n5 = await Note.save({ title: 'Just minding my own business' });
|
||||
|
||||
await engine.syncTables();
|
||||
rows = await engine.search('demand', { fuzzy: false });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].id).toBe(n1.id);
|
||||
|
||||
|
||||
rows = await engine.search('demand', { fuzzy: true });
|
||||
expect(rows.length).toBe(3);
|
||||
expect(rows[0].id).toBe(n1.id);
|
||||
expect(rows[1].id).toBe(n3.id);
|
||||
expect(rows[2].id).toBe(n2.id);
|
||||
}));
|
||||
|
||||
it('should consider any:1', asyncTest(async () => {
|
||||
let rows;
|
||||
const n1 = await Note.save({ title: 'cat' });
|
||||
const n2 = await Note.save({ title: 'cats' });
|
||||
const n3 = await Note.save({ title: 'cot' });
|
||||
|
||||
const n4 = await Note.save({ title: 'defenestrate' });
|
||||
const n5 = await Note.save({ title: 'defenstrate' });
|
||||
const n6 = await Note.save({ title: 'defenestrated' });
|
||||
|
||||
const n7 = await Note.save({ title: 'he defenestrated the cat' });
|
||||
|
||||
await engine.syncTables();
|
||||
|
||||
rows = await engine.search('defenestrated cat', { fuzzy: true });
|
||||
expect(rows.length).toBe(1);
|
||||
|
||||
rows = await engine.search('any:1 defenestrated cat', { fuzzy: true });
|
||||
expect(rows.length).toBe(7);
|
||||
}));
|
||||
|
||||
it('should leave phrase searches alone', asyncTest(async () => {
|
||||
let rows;
|
||||
const n1 = await Note.save({ title: 'abc def' });
|
||||
const n2 = await Note.save({ title: 'def ghi' });
|
||||
const n3 = await Note.save({ title: 'ghi jkl' });
|
||||
const n4 = await Note.save({ title: 'def abc' });
|
||||
const n5 = await Note.save({ title: 'mno pqr ghi jkl' });
|
||||
|
||||
await engine.syncTables();
|
||||
|
||||
rows = await engine.search('abc def', { fuzzy: true });
|
||||
expect(rows.length).toBe(2);
|
||||
expect(rows.map(r=>r.id)).toContain(n1.id);
|
||||
expect(rows.map(r=>r.id)).toContain(n4.id);
|
||||
|
||||
rows = await engine.search('"abc def"', { fuzzy: true });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows.map(r=>r.id)).toContain(n1.id);
|
||||
|
||||
rows = await engine.search('"ghi jkl"', { fuzzy: true });
|
||||
expect(rows.length).toBe(2);
|
||||
expect(rows.map(r=>r.id)).toContain(n3.id);
|
||||
expect(rows.map(r=>r.id)).toContain(n5.id);
|
||||
|
||||
rows = await engine.search('"ghi jkl" mno', { fuzzy: true });
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows.map(r=>r.id)).toContain(n5.id);
|
||||
|
||||
rows = await engine.search('any:1 "ghi jkl" mno', { fuzzy: true });
|
||||
expect(rows.length).toBe(2);
|
||||
expect(rows.map(r=>r.id)).toContain(n3.id);
|
||||
expect(rows.map(r=>r.id)).toContain(n5.id);
|
||||
}));
|
||||
|
||||
|
||||
});
|
@ -4,6 +4,7 @@ const { themeStyle } = require('lib/theme');
|
||||
const { _ } = require('lib/locale.js');
|
||||
const { bridge } = require('electron').remote.require('./bridge');
|
||||
const CommandService = require('lib/services/CommandService').default;
|
||||
const Setting = require('lib/models/Setting.js');
|
||||
|
||||
const commands = [
|
||||
require('./commands/focusSearch'),
|
||||
@ -28,7 +29,7 @@ class HeaderComponent extends React.Component {
|
||||
|
||||
const triggerOnQuery = query => {
|
||||
clearTimeout(this.scheduleSearchChangeEventIid_);
|
||||
if (this.searchOnQuery_) this.searchOnQuery_(query);
|
||||
if (this.searchOnQuery_) this.searchOnQuery_(query, Setting.value('db.fuzzySearchEnabled'));
|
||||
this.scheduleSearchChangeEventIid_ = null;
|
||||
};
|
||||
|
||||
|
@ -445,8 +445,8 @@ class MainScreenComponent extends React.Component {
|
||||
headerItems.push({
|
||||
title: _('Search...'),
|
||||
iconName: 'fa-search',
|
||||
onQuery: query => {
|
||||
CommandService.instance().execute('search', { query });
|
||||
onQuery: (query, fuzzy = false) => {
|
||||
CommandService.instance().execute('search', { query, fuzzy });
|
||||
},
|
||||
type: 'search',
|
||||
});
|
||||
|
@ -10,7 +10,7 @@ export const declaration:CommandDeclaration = {
|
||||
|
||||
export const runtime = (comp:any):CommandRuntime => {
|
||||
return {
|
||||
execute: async ({ query }:any) => {
|
||||
execute: async ({ query, fuzzy }:any) => {
|
||||
console.info('RUNTIME', query);
|
||||
|
||||
if (!comp.searchId_) comp.searchId_ = uuid.create();
|
||||
@ -23,6 +23,7 @@ export const runtime = (comp:any):CommandRuntime => {
|
||||
query_pattern: query,
|
||||
query_folder_id: null,
|
||||
type_: BaseModel.TYPE_SEARCH,
|
||||
fuzzy: fuzzy,
|
||||
},
|
||||
});
|
||||
|
||||
|
@ -365,7 +365,7 @@ function NoteEditor(props: NoteEditorProps) {
|
||||
);
|
||||
}
|
||||
|
||||
const searchMarkers = useSearchMarkers(showLocalSearch, localSearchMarkerOptions, props.searches, props.selectedSearchId);
|
||||
const searchMarkers = useSearchMarkers(showLocalSearch, localSearchMarkerOptions, props.searches, props.selectedSearchId, props.highlightedWords);
|
||||
|
||||
const editorProps:NoteBodyEditorProps = {
|
||||
ref: editorRef,
|
||||
@ -531,6 +531,7 @@ const mapStateToProps = (state: any) => {
|
||||
customCss: state.customCss,
|
||||
noteVisiblePanes: state.noteVisiblePanes,
|
||||
watchedResources: state.watchedResources,
|
||||
highlightedWords: state.highlightedWords,
|
||||
};
|
||||
};
|
||||
|
||||
|
@ -23,6 +23,7 @@ export interface NoteEditorProps {
|
||||
customCss: string,
|
||||
noteVisiblePanes: string[],
|
||||
watchedResources: any,
|
||||
highlightedWords: any[],
|
||||
}
|
||||
|
||||
export interface NoteBodyEditorProps {
|
||||
|
@ -1,8 +1,5 @@
|
||||
import { useMemo } from 'react';
|
||||
|
||||
const BaseModel = require('lib/BaseModel.js');
|
||||
const SearchEngine = require('lib/services/searchengine/SearchEngine');
|
||||
|
||||
interface SearchMarkersOptions {
|
||||
searchTimestamp: number,
|
||||
selectedIndex: number,
|
||||
@ -25,18 +22,14 @@ function defaultSearchMarkers():SearchMarkers {
|
||||
};
|
||||
}
|
||||
|
||||
export default function useSearchMarkers(showLocalSearch:boolean, localSearchMarkerOptions:Function, searches:any[], selectedSearchId:string) {
|
||||
|
||||
export default function useSearchMarkers(showLocalSearch:boolean, localSearchMarkerOptions:Function, searches:any[], selectedSearchId:string, highlightedWords: any[] = []) {
|
||||
return useMemo(():SearchMarkers => {
|
||||
if (showLocalSearch) return localSearchMarkerOptions();
|
||||
|
||||
const output = defaultSearchMarkers();
|
||||
|
||||
const search = BaseModel.byId(searches, selectedSearchId);
|
||||
if (search) {
|
||||
const parsedQuery = SearchEngine.instance().parseQuery(search.query_pattern);
|
||||
output.keywords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
|
||||
}
|
||||
output.keywords = highlightedWords;
|
||||
|
||||
return output;
|
||||
}, [showLocalSearch, localSearchMarkerOptions, searches, selectedSearchId]);
|
||||
}, [highlightedWords, showLocalSearch, localSearchMarkerOptions, searches, selectedSearchId]);
|
||||
}
|
||||
|
@ -7,7 +7,6 @@ const BaseModel = require('lib/BaseModel');
|
||||
const { _ } = require('lib/locale.js');
|
||||
const { bridge } = require('electron').remote.require('./bridge');
|
||||
const eventManager = require('lib/eventManager');
|
||||
const SearchEngine = require('lib/services/searchengine/SearchEngine');
|
||||
const Note = require('lib/models/Note');
|
||||
const Setting = require('lib/models/Setting');
|
||||
const NoteListUtils = require('../utils/NoteListUtils');
|
||||
@ -229,8 +228,7 @@ class NoteListComponent extends React.Component {
|
||||
if (this.props.notesParentType === 'Search') {
|
||||
const query = BaseModel.byId(this.props.searches, this.props.selectedSearchId);
|
||||
if (query) {
|
||||
const parsedQuery = SearchEngine.instance().parseQuery(query.query_pattern);
|
||||
return SearchEngine.instance().allParsedQueryTerms(parsedQuery);
|
||||
return this.props.highlightedWords;
|
||||
}
|
||||
}
|
||||
return [];
|
||||
@ -460,6 +458,7 @@ const mapStateToProps = state => {
|
||||
provisionalNoteIds: state.provisionalNoteIds,
|
||||
isInsertingNotes: state.isInsertingNotes,
|
||||
noteSortOrder: state.settings['notes.sortOrder.field'],
|
||||
highlightedWords: state.highlightedWords,
|
||||
};
|
||||
};
|
||||
|
||||
|
@ -17,6 +17,9 @@ const tasks = {
|
||||
electronRebuild: {
|
||||
fn: require('./tools/electronRebuild.js'),
|
||||
},
|
||||
compileExtensions: {
|
||||
fn: require('../Tools/gulp/tasks/compileExtensions.js'),
|
||||
},
|
||||
copyLib: require('../Tools/gulp/tasks/copyLib'),
|
||||
tsc: require('../Tools/gulp/tasks/tsc'),
|
||||
updateIgnoredTypeScriptBuild: require('../Tools/gulp/tasks/updateIgnoredTypeScriptBuild'),
|
||||
@ -25,6 +28,7 @@ const tasks = {
|
||||
utils.registerGulpTasks(gulp, tasks);
|
||||
|
||||
const buildSeries = [
|
||||
'compileExtensions',
|
||||
'copyLib',
|
||||
];
|
||||
|
||||
|
266
ElectronClient/package-lock.json
generated
266
ElectronClient/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -51,6 +51,10 @@
|
||||
{
|
||||
"from": "build-win/Joplin.VisualElementsManifest.xml",
|
||||
"to": "."
|
||||
},
|
||||
{
|
||||
"from": "lib/sql-extensions/spellfix.dll",
|
||||
"to": "usr/lib/spellfix.dll"
|
||||
}
|
||||
],
|
||||
"extraResources": [
|
||||
@ -65,7 +69,13 @@
|
||||
"artifactName": "${productName}Portable.${ext}"
|
||||
},
|
||||
"mac": {
|
||||
"icon": "../../Assets/macOs.icns"
|
||||
"icon": "../../Assets/macOs.icns",
|
||||
"extraFiles": [
|
||||
{
|
||||
"from": "lib/sql-extensions/spellfix.dylib",
|
||||
"to": "usr/lib/spellfix.dylib"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux": {
|
||||
"icon": "../Assets/LinuxIcons/256x256.png",
|
||||
@ -73,7 +83,13 @@
|
||||
"desktop": {
|
||||
"Icon": "joplin"
|
||||
},
|
||||
"target": "AppImage"
|
||||
"target": "AppImage",
|
||||
"extraFiles": [
|
||||
{
|
||||
"from": "lib/sql-extensions/spellfix.so",
|
||||
"to": "usr/lib/spellfix.so"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"homepage": "https://github.com/laurent22/joplin#readme",
|
||||
|
@ -177,9 +177,8 @@ class Dialog extends React.PureComponent {
|
||||
return output.join(' ');
|
||||
}
|
||||
|
||||
keywords(searchQuery) {
|
||||
const parsedQuery = SearchEngine.instance().parseQuery(searchQuery);
|
||||
return SearchEngine.instance().allParsedQueryTerms(parsedQuery);
|
||||
keywords() {
|
||||
return this.props.highlightedWords;
|
||||
}
|
||||
|
||||
markupToHtml() {
|
||||
@ -227,7 +226,7 @@ class Dialog extends React.PureComponent {
|
||||
}
|
||||
} else {
|
||||
const limit = 20;
|
||||
const searchKeywords = this.keywords(searchQuery);
|
||||
const searchKeywords = this.keywords();
|
||||
const notes = await Note.byIds(results.map(result => result.id).slice(0, limit), { fields: ['id', 'body', 'markup_language', 'is_todo', 'todo_completed'] });
|
||||
const notesById = notes.reduce((obj, { id, body, markup_language }) => ((obj[[id]] = { id, body, markup_language }), obj), {});
|
||||
|
||||
@ -283,7 +282,7 @@ class Dialog extends React.PureComponent {
|
||||
this.setState({
|
||||
listType: listType,
|
||||
results: results,
|
||||
keywords: this.keywords(searchQuery),
|
||||
keywords: this.keywords(),
|
||||
selectedItemId: results.length === 0 ? null : results[0].id,
|
||||
resultsInBody: resultsInBody,
|
||||
});
|
||||
@ -455,6 +454,7 @@ const mapStateToProps = (state) => {
|
||||
folders: state.folders,
|
||||
theme: state.settings.theme,
|
||||
showCompletedTodos: state.settings.showCompletedTodos,
|
||||
highlightedWords: state.highlightedWords,
|
||||
};
|
||||
};
|
||||
|
||||
|
@ -280,6 +280,7 @@ class BaseApplication {
|
||||
});
|
||||
|
||||
let notes = [];
|
||||
let highlightedWords = [];
|
||||
|
||||
if (parentId) {
|
||||
if (parentType === Folder.modelType()) {
|
||||
@ -288,12 +289,21 @@ class BaseApplication {
|
||||
notes = await Tag.notes(parentId, options);
|
||||
} else if (parentType === BaseModel.TYPE_SEARCH) {
|
||||
const search = BaseModel.byId(state.searches, parentId);
|
||||
notes = await SearchEngineUtils.notesForQuery(search.query_pattern);
|
||||
notes = await SearchEngineUtils.notesForQuery(search.query_pattern, { fuzzy: search.fuzzy });
|
||||
const parsedQuery = await SearchEngine.instance().parseQuery(search.query_pattern, search.fuzzy);
|
||||
highlightedWords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
|
||||
} else if (parentType === BaseModel.TYPE_SMART_FILTER) {
|
||||
notes = await Note.previews(parentId, options);
|
||||
}
|
||||
}
|
||||
|
||||
if (highlightedWords.length) {
|
||||
this.store().dispatch({
|
||||
type: 'SET_HIGHLIGHTED',
|
||||
words: highlightedWords,
|
||||
});
|
||||
}
|
||||
|
||||
this.store().dispatch({
|
||||
type: 'NOTE_UPDATE_ALL',
|
||||
notes: notes,
|
||||
@ -681,6 +691,23 @@ class BaseApplication {
|
||||
this.database_ = new JoplinDatabase(new DatabaseDriverNode());
|
||||
this.database_.setLogExcludedQueryTypes(['SELECT']);
|
||||
this.database_.setLogger(this.dbLogger_);
|
||||
|
||||
if (Setting.value('env') === 'dev') {
|
||||
if (shim.isElectron()) {
|
||||
this.database_.extensionToLoad = './lib/sql-extensions/spellfix';
|
||||
}
|
||||
} else {
|
||||
if (shim.isElectron()) {
|
||||
if (shim.isWindows()) {
|
||||
const appDir = process.execPath.substring(0, process.execPath.lastIndexOf('\\'));
|
||||
this.database_.extensionToLoad = `${appDir}/usr/lib/spellfix`;
|
||||
} else {
|
||||
const appDir = process.execPath.substring(0, process.execPath.lastIndexOf('/'));
|
||||
this.database_.extensionToLoad = `${appDir}/usr/lib/spellfix`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await this.database_.open({ name: `${profileDir}/database.sqlite` });
|
||||
|
||||
// if (Setting.value('env') === 'dev') await this.database_.clearForTesting();
|
||||
@ -707,6 +734,11 @@ class BaseApplication {
|
||||
setLocale(Setting.value('locale'));
|
||||
}
|
||||
|
||||
if (Setting.value('db.fuzzySearchEnabled') === -1) {
|
||||
const fuzzySearchEnabled = await this.database_.fuzzySearchEnabled();
|
||||
Setting.setValue('db.fuzzySearchEnabled', fuzzySearchEnabled ? 1 : 0);
|
||||
}
|
||||
|
||||
if (Setting.value('encryption.shouldReencrypt') < 0) {
|
||||
// We suggest re-encryption if the user has at least one notebook
|
||||
// and if encryption is enabled. This code runs only when shouldReencrypt = -1
|
||||
|
@ -40,7 +40,6 @@ const ImagePicker = require('react-native-image-picker');
|
||||
const { SelectDateTimeDialog } = require('lib/components/select-date-time-dialog.js');
|
||||
const ShareExtension = require('lib/ShareExtension.js').default;
|
||||
const CameraView = require('lib/components/CameraView');
|
||||
const SearchEngine = require('lib/services/searchengine/SearchEngine');
|
||||
const urlUtils = require('lib/urlUtils');
|
||||
|
||||
class NoteScreenComponent extends BaseScreenComponent {
|
||||
@ -975,8 +974,7 @@ class NoteScreenComponent extends BaseScreenComponent {
|
||||
// Currently keyword highlighting is supported only when FTS is available.
|
||||
let keywords = [];
|
||||
if (this.props.searchQuery && !!this.props.ftsEnabled) {
|
||||
const parsedQuery = SearchEngine.instance().parseQuery(this.props.searchQuery);
|
||||
keywords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
|
||||
keywords = this.props.highlightedWords;
|
||||
}
|
||||
|
||||
// Note: as of 2018-12-29 it's important not to display the viewer if the note body is empty,
|
||||
@ -1016,8 +1014,7 @@ class NoteScreenComponent extends BaseScreenComponent {
|
||||
// Currently keyword highlighting is supported only when FTS is available.
|
||||
let keywords = [];
|
||||
if (this.props.searchQuery && !!this.props.ftsEnabled) {
|
||||
const parsedQuery = SearchEngine.instance().parseQuery(this.props.searchQuery);
|
||||
keywords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
|
||||
keywords = this.props.highlightedWords;
|
||||
}
|
||||
|
||||
const onCheckboxChange = newBody => {
|
||||
@ -1202,6 +1199,7 @@ const NoteScreen = connect(state => {
|
||||
sharedData: state.sharedData,
|
||||
showSideMenu: state.showSideMenu,
|
||||
provisionalNoteIds: state.provisionalNoteIds,
|
||||
highlightedWords: state.highlightedWords,
|
||||
};
|
||||
})(NoteScreenComponent);
|
||||
|
||||
|
@ -36,6 +36,10 @@ class DatabaseDriverNode {
|
||||
});
|
||||
}
|
||||
|
||||
loadExtension(path) {
|
||||
return this.db_.loadExtension(path);
|
||||
}
|
||||
|
||||
selectAll(sql, params = null) {
|
||||
if (!params) params = {};
|
||||
return new Promise((resolve, reject) => {
|
||||
|
@ -50,6 +50,10 @@ class DatabaseDriverReactNative {
|
||||
});
|
||||
}
|
||||
|
||||
loadExtension(path) {
|
||||
throw new Error(`No extension support for ${path} in react-native-sqlite-storage`);
|
||||
}
|
||||
|
||||
exec(sql, params = null) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db_.executeSql(
|
||||
|
@ -97,6 +97,16 @@ class Database {
|
||||
return this.tryCall('selectOne', sql, params);
|
||||
}
|
||||
|
||||
async loadExtension(path) {
|
||||
let result = null;
|
||||
try {
|
||||
result = await this.driver().loadExtension(path);
|
||||
return result;
|
||||
} catch (e) {
|
||||
throw new Error(`Could not load extension ${path}`);
|
||||
}
|
||||
}
|
||||
|
||||
async selectAll(sql, params = null) {
|
||||
return this.tryCall('selectAll', sql, params);
|
||||
}
|
||||
|
@ -126,6 +126,7 @@ class JoplinDatabase extends Database {
|
||||
this.tableFields_ = null;
|
||||
this.version_ = null;
|
||||
this.tableFieldNames_ = {};
|
||||
this.extensionToLoad = './build/lib/sql-extensions/spellfix';
|
||||
this.eventEmitter_ = new EventEmitter();
|
||||
}
|
||||
|
||||
@ -283,6 +284,8 @@ class JoplinDatabase extends Database {
|
||||
if (tableName == 'table_fields') continue;
|
||||
if (tableName == 'sqlite_sequence') continue;
|
||||
if (tableName.indexOf('notes_fts') === 0) continue;
|
||||
if (tableName == 'notes_spellfix') continue;
|
||||
if (tableName == 'search_aux') continue;
|
||||
chain.push(() => {
|
||||
return this.selectAll(`PRAGMA table_info("${tableName}")`).then(pragmas => {
|
||||
for (let i = 0; i < pragmas.length; i++) {
|
||||
@ -332,7 +335,7 @@ class JoplinDatabase extends Database {
|
||||
// must be set in the synchronizer too.
|
||||
|
||||
// Note: v16 and v17 don't do anything. They were used to debug an issue.
|
||||
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33];
|
||||
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34];
|
||||
|
||||
let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
|
||||
|
||||
@ -848,13 +851,20 @@ class JoplinDatabase extends Database {
|
||||
queries.push(this.addMigrationFile(33));
|
||||
}
|
||||
|
||||
if (targetVersion == 34) {
|
||||
queries.push('CREATE VIRTUAL TABLE search_aux USING fts4aux(notes_fts)');
|
||||
queries.push('CREATE VIRTUAL TABLE notes_spellfix USING spellfix1');
|
||||
}
|
||||
|
||||
queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });
|
||||
|
||||
try {
|
||||
await this.transactionExecBatch(queries);
|
||||
} catch (error) {
|
||||
if (targetVersion === 15 || targetVersion === 18 || targetVersion === 33) {
|
||||
this.logger().warn('Could not upgrade to database v15 or v18 or v33- FTS feature will not be used', error);
|
||||
this.logger().warn('Could not upgrade to database v15 or v18 or v33 - FTS feature will not be used', error);
|
||||
} else if (targetVersion === 34) {
|
||||
this.logger().warn('Could not upgrade to database v34 - fuzzy search will not be used', error);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
@ -881,6 +891,17 @@ class JoplinDatabase extends Database {
|
||||
return true;
|
||||
}
|
||||
|
||||
async fuzzySearchEnabled() {
|
||||
try {
|
||||
await this.selectOne('SELECT count(*) FROM notes_spellfix');
|
||||
} catch (error) {
|
||||
this.logger().warn('Fuzzy search check failed', error);
|
||||
return false;
|
||||
}
|
||||
this.logger().info('Fuzzy search check succeeded');
|
||||
return true;
|
||||
}
|
||||
|
||||
version() {
|
||||
return this.version_;
|
||||
}
|
||||
@ -888,6 +909,12 @@ class JoplinDatabase extends Database {
|
||||
async initialize() {
|
||||
this.logger().info('Checking for database schema update...');
|
||||
|
||||
try {
|
||||
await this.loadExtension(this.extensionToLoad);
|
||||
} catch (error) {
|
||||
console.info(error);
|
||||
}
|
||||
|
||||
let versionRow = null;
|
||||
try {
|
||||
// Will throw if the database has not been created yet, but this is handled below
|
||||
|
@ -506,6 +506,7 @@ class Setting extends BaseModel {
|
||||
|
||||
'keychain.supported': { value: -1, type: Setting.TYPE_INT, public: false },
|
||||
'db.ftsEnabled': { value: -1, type: Setting.TYPE_INT, public: false },
|
||||
'db.fuzzySearchEnabled': { value: -1, type: Setting.TYPE_INT, public: false },
|
||||
'encryption.enabled': { value: false, type: Setting.TYPE_BOOL, public: false },
|
||||
'encryption.activeMasterKeyId': { value: '', type: Setting.TYPE_STRING, public: false },
|
||||
'encryption.passwordCache': { value: {}, type: Setting.TYPE_OBJECT, public: false, secure: true },
|
||||
|
@ -12,6 +12,7 @@ const defaultState = {
|
||||
masterKeys: [],
|
||||
notLoadedMasterKeys: [],
|
||||
searches: [],
|
||||
highlightedWords: [],
|
||||
selectedNoteIds: [],
|
||||
selectedNoteHash: '',
|
||||
selectedFolderId: null,
|
||||
@ -946,6 +947,9 @@ const reducer = (state = defaultState, action) => {
|
||||
}
|
||||
newState.selectedNoteIds = [];
|
||||
break;
|
||||
case 'SET_HIGHLIGHTED':
|
||||
newState = Object.assign({}, state, { highlightedWords: action.words });
|
||||
break;
|
||||
|
||||
case 'APP_STATE_SET':
|
||||
newState = Object.assign({}, state);
|
||||
|
@ -81,6 +81,11 @@ class SearchEngine {
|
||||
);
|
||||
}
|
||||
|
||||
if (!noteIds.length && (Setting.value('db.fuzzySearchEnabled') == 1)) {
|
||||
// On the last loop
|
||||
queries.push({ sql: 'INSERT INTO notes_spellfix(word,rank) SELECT term, documents FROM search_aux WHERE col=\'*\'' });
|
||||
}
|
||||
|
||||
await this.db().transactionExecBatch(queries);
|
||||
}
|
||||
|
||||
@ -145,7 +150,16 @@ class SearchEngine {
|
||||
[BaseModel.TYPE_NOTE, lastChangeId]
|
||||
);
|
||||
|
||||
if (!changes.length) break;
|
||||
const queries = [];
|
||||
|
||||
if (!changes.length) {
|
||||
if (Setting.value('db.fuzzySearchEnabled') === 1) {
|
||||
queries.push({ sql: 'DELETE FROM notes_spellfix' });
|
||||
queries.push({ sql: 'INSERT INTO notes_spellfix(word,rank) SELECT term, documents FROM search_aux WHERE col=\'*\'' });
|
||||
await this.db().transactionExecBatch(queries);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
const noteIds = changes.map(a => a.item_id);
|
||||
const notes = await Note.modelSelectAll(`
|
||||
@ -153,8 +167,6 @@ class SearchEngine {
|
||||
FROM notes WHERE id IN ("${noteIds.join('","')}") AND is_conflict = 0 AND encryption_applied = 0`
|
||||
);
|
||||
|
||||
const queries = [];
|
||||
|
||||
for (let i = 0; i < changes.length; i++) {
|
||||
const change = changes[i];
|
||||
|
||||
@ -254,7 +266,7 @@ class SearchEngine {
|
||||
|
||||
|
||||
|
||||
calculateWeightBM25_(rows) {
|
||||
calculateWeightBM25_(rows, fuzzyScore) {
|
||||
// https://www.sqlite.org/fts3.html#matchinfo
|
||||
// pcnalx are the arguments passed to matchinfo
|
||||
// p - The number of matchable phrases in the query.
|
||||
@ -318,14 +330,20 @@ class SearchEngine {
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const row = rows[i];
|
||||
row.weight = 0;
|
||||
row.fuzziness = 1000;
|
||||
row.wordFound = [];
|
||||
for (let j = 0; j < numPhrases; j++) {
|
||||
let found = false;
|
||||
columns.forEach(column => {
|
||||
const rowsWithHits = docsWithHits(X[i], column, j);
|
||||
const frequencyHits = hitsThisRow(X[i], column, j);
|
||||
|
||||
const idf = IDF(rowsWithHits, numRows);
|
||||
found = found ? found : (frequencyHits > 0);
|
||||
|
||||
row.weight += BM25(idf, frequencyHits, numTokens[column][i], avgTokens[column]);
|
||||
row.fuzziness = (frequencyHits > 0) ? Math.min(row.fuzziness, fuzzyScore[j]) : row.fuzziness;
|
||||
});
|
||||
row.wordFound.push(found);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -345,23 +363,40 @@ class SearchEngine {
|
||||
|
||||
row.fields = Object.keys(matchedFields).filter(key => matchedFields[key]);
|
||||
row.weight = 0;
|
||||
row.fuzziness = 0;
|
||||
}
|
||||
}
|
||||
|
||||
processResults_(rows, parsedQuery, isBasicSearchResults = false) {
|
||||
const rowContainsAllWords = (wordsFound, numFuzzyMatches) => {
|
||||
let start = 0;
|
||||
let end = 0;
|
||||
for (let i = 0; i < numFuzzyMatches.length; i++) {
|
||||
end = end + numFuzzyMatches[i];
|
||||
if (!(wordsFound.slice(start, end).find(x => x))) {
|
||||
// This note doesn't contain any fuzzy matches for the word
|
||||
return false;
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
if (isBasicSearchResults) {
|
||||
this.processBasicSearchResults_(rows, parsedQuery);
|
||||
} else {
|
||||
this.calculateWeightBM25_(rows);
|
||||
this.calculateWeightBM25_(rows, parsedQuery.fuzzyScore);
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const row = rows[i];
|
||||
row.include = (parsedQuery.fuzzy && !parsedQuery.any) ? rowContainsAllWords(row.wordFound, parsedQuery.numFuzzyMatches) : true;
|
||||
const offsets = row.offsets.split(' ').map(o => Number(o));
|
||||
row.fields = this.fieldNamesFromOffsets_(offsets);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
rows.sort((a, b) => {
|
||||
if (a.fuzziness < b.fuzziness) return -1;
|
||||
if (a.fuzziness > b.fuzziness) return +1;
|
||||
if (a.fields.includes('title') && !b.fields.includes('title')) return -1;
|
||||
if (!a.fields.includes('title') && b.fields.includes('title')) return +1;
|
||||
if (a.weight < b.weight) return +1;
|
||||
@ -389,21 +424,75 @@ class SearchEngine {
|
||||
return regexString;
|
||||
}
|
||||
|
||||
parseQuery(query) {
|
||||
async fuzzifier(words) {
|
||||
const fuzzyMatches = [];
|
||||
words.forEach(word => {
|
||||
const fuzzyWords = this.db().selectAll('SELECT word, score FROM notes_spellfix WHERE word MATCH ? AND top=3', [word]);
|
||||
fuzzyMatches.push(fuzzyWords);
|
||||
});
|
||||
return await Promise.all(fuzzyMatches);
|
||||
}
|
||||
|
||||
async parseQuery(query, fuzzy = false) {
|
||||
// fuzzy = false;
|
||||
const trimQuotes = (str) => str.startsWith('"') ? str.substr(1, str.length - 2) : str;
|
||||
|
||||
let allTerms = [];
|
||||
let allFuzzyTerms = [];
|
||||
|
||||
try {
|
||||
allTerms = filterParser(query);
|
||||
} catch (error) {
|
||||
console.warn(error);
|
||||
}
|
||||
|
||||
const textTerms = allTerms.filter(x => x.name === 'text').map(x => trimQuotes(x.value));
|
||||
const titleTerms = allTerms.filter(x => x.name === 'title').map(x => trimQuotes(x.value));
|
||||
const bodyTerms = allTerms.filter(x => x.name === 'body').map(x => trimQuotes(x.value));
|
||||
const textTerms = allTerms.filter(x => x.name === 'text' && !x.negated);
|
||||
const titleTerms = allTerms.filter(x => x.name === 'title' && !x.negated);
|
||||
const bodyTerms = allTerms.filter(x => x.name === 'body' && !x.negated);
|
||||
|
||||
const terms = { _: textTerms, 'title': titleTerms, 'body': bodyTerms };
|
||||
const fuzzyScore = [];
|
||||
let numFuzzyMatches = [];
|
||||
let terms = null;
|
||||
if (fuzzy) {
|
||||
const fuzzyText = await this.fuzzifier(textTerms.filter(x => !x.quoted).map(x => trimQuotes(x.value)));
|
||||
const fuzzyTitle = await this.fuzzifier(titleTerms.map(x => trimQuotes(x.value)));
|
||||
const fuzzyBody = await this.fuzzifier(bodyTerms.map(x => trimQuotes(x.value)));
|
||||
const phraseSearches = textTerms.filter(x => x.quoted).map(x => x.value);
|
||||
|
||||
// Save number of matches we got for each word
|
||||
// fuzzifier() is currently set to return at most 3 matches)
|
||||
// We need to know which fuzzy words go together so that we can filter out notes that don't contain a required word.
|
||||
numFuzzyMatches = fuzzyText.concat(fuzzyTitle).concat(fuzzyBody).map(x => x.length);
|
||||
for (let i = 0; i < phraseSearches.length; i++) {
|
||||
numFuzzyMatches.push(1); // Phrase searches are preserved without fuzzification
|
||||
}
|
||||
|
||||
const mergedFuzzyText = [].concat.apply([], fuzzyText);
|
||||
const mergedFuzzyTitle = [].concat.apply([], fuzzyTitle);
|
||||
const mergedFuzzyBody = [].concat.apply([], fuzzyBody);
|
||||
|
||||
const fuzzyTextTerms = mergedFuzzyText.map(x => { return { name: 'text', value: x.word, negated: false, score: x.score }; });
|
||||
const fuzzyTitleTerms = mergedFuzzyTitle.map(x => { return { name: 'title', value: x.word, negated: false, score: x.score }; });
|
||||
const fuzzyBodyTerms = mergedFuzzyBody.map(x => { return { name: 'body', value: x.word, negated: false, score: x.score }; });
|
||||
const phraseTextTerms = phraseSearches.map(x => { return { name: 'text', value: x, negated: false, score: 0 }; });
|
||||
|
||||
allTerms = allTerms.filter(x => (x.name !== 'text' && x.name !== 'title' && x.name !== 'body'));
|
||||
|
||||
allFuzzyTerms = allTerms.concat(fuzzyTextTerms).concat(fuzzyTitleTerms).concat(fuzzyBodyTerms).concat(phraseTextTerms);
|
||||
|
||||
const allTextTerms = allFuzzyTerms.filter(x => x.name === 'title' || x.name === 'body' || x.name === 'text');
|
||||
for (let i = 0; i < allTextTerms.length; i++) {
|
||||
fuzzyScore.push(allFuzzyTerms[i].score ? allFuzzyTerms[i].score : 0);
|
||||
}
|
||||
|
||||
terms = { _: fuzzyTextTerms.concat(phraseTextTerms).map(x =>trimQuotes(x.value)), 'title': fuzzyTitleTerms.map(x =>trimQuotes(x.value)), 'body': fuzzyBodyTerms.map(x =>trimQuotes(x.value)) };
|
||||
} else {
|
||||
const nonNegatedTextTerms = textTerms.length + titleTerms.length + bodyTerms.length;
|
||||
for (let i = 0; i < nonNegatedTextTerms; i++) {
|
||||
fuzzyScore.push(0);
|
||||
}
|
||||
terms = { _: textTerms.map(x =>trimQuotes(x.value)), 'title': titleTerms.map(x =>trimQuotes(x.value)), 'body': bodyTerms.map(x =>trimQuotes(x.value)) };
|
||||
}
|
||||
|
||||
// Filter terms:
|
||||
// - Convert wildcards to regex
|
||||
@ -445,7 +534,11 @@ class SearchEngine {
|
||||
termCount: termCount,
|
||||
keys: keys,
|
||||
terms: terms, // text terms
|
||||
allTerms: allTerms,
|
||||
allTerms: fuzzy ? allFuzzyTerms : allTerms,
|
||||
fuzzyScore: fuzzyScore,
|
||||
numFuzzyMatches: numFuzzyMatches,
|
||||
fuzzy: fuzzy,
|
||||
any: !!allTerms.find(term => term.name === 'any'),
|
||||
};
|
||||
}
|
||||
|
||||
@ -474,7 +567,7 @@ class SearchEngine {
|
||||
|
||||
async basicSearch(query) {
|
||||
query = query.replace(/\*/, '');
|
||||
const parsedQuery = this.parseQuery(query);
|
||||
const parsedQuery = await this.parseQuery(query);
|
||||
const searchOptions = {};
|
||||
|
||||
for (const key of parsedQuery.keys) {
|
||||
@ -489,8 +582,8 @@ class SearchEngine {
|
||||
return Note.previews(null, searchOptions);
|
||||
}
|
||||
|
||||
determineSearchType_(query, preferredSearchType) {
|
||||
if (preferredSearchType === SearchEngine.SEARCH_TYPE_BASIC) return SearchEngine.SEARCH_TYPE_BASIC;
|
||||
determineSearchType_(query, options) {
|
||||
if (options.searchType === SearchEngine.SEARCH_TYPE_BASIC) return SearchEngine.SEARCH_TYPE_BASIC;
|
||||
|
||||
// If preferredSearchType is "fts" we auto-detect anyway
|
||||
// because it's not always supported.
|
||||
@ -499,9 +592,12 @@ class SearchEngine {
|
||||
|
||||
if (!Setting.value('db.ftsEnabled') || ['ja', 'zh', 'ko', 'th'].indexOf(st) >= 0) {
|
||||
return SearchEngine.SEARCH_TYPE_BASIC;
|
||||
} else if ((Setting.value('db.fuzzySearchEnabled') === 1) && options.fuzzy) {
|
||||
return SearchEngine.SEARCH_TYPE_FTS_FUZZY;
|
||||
} else {
|
||||
return SearchEngine.SEARCH_TYPE_FTS;
|
||||
}
|
||||
|
||||
return SearchEngine.SEARCH_TYPE_FTS;
|
||||
}
|
||||
|
||||
async search(searchString, options = null) {
|
||||
@ -511,28 +607,31 @@ class SearchEngine {
|
||||
|
||||
searchString = this.normalizeText_(searchString);
|
||||
|
||||
const searchType = this.determineSearchType_(searchString, options.searchType);
|
||||
const searchType = this.determineSearchType_(searchString, options);
|
||||
|
||||
if (searchType === SearchEngine.SEARCH_TYPE_BASIC) {
|
||||
// Non-alphabetical languages aren't support by SQLite FTS (except with extensions which are not available in all platforms)
|
||||
const rows = await this.basicSearch(searchString);
|
||||
const parsedQuery = this.parseQuery(searchString);
|
||||
const parsedQuery = await this.parseQuery(searchString);
|
||||
this.processResults_(rows, parsedQuery, true);
|
||||
return rows;
|
||||
} else {
|
||||
// SEARCH_TYPE_FTS
|
||||
// SEARCH_TYPE_FTS or SEARCH_TYPE_FTS_FUZZY
|
||||
// FTS will ignore all special characters, like "-" in the index. So if
|
||||
// we search for "this-phrase" it won't find it because it will only
|
||||
// see "this phrase" in the index. Because of this, we remove the dashes
|
||||
// when searching.
|
||||
// https://github.com/laurent22/joplin/issues/1075#issuecomment-459258856
|
||||
|
||||
const parsedQuery = this.parseQuery(searchString);
|
||||
const parsedQuery = await this.parseQuery(searchString, options.fuzzy);
|
||||
|
||||
try {
|
||||
const { query, params } = queryBuilder(parsedQuery.allTerms);
|
||||
const { query, params } = (searchType === SearchEngine.SEARCH_TYPE_FTS_FUZZY) ? queryBuilder(parsedQuery.allTerms, true) : queryBuilder(parsedQuery.allTerms, false);
|
||||
const rows = await this.db().selectAll(query, params);
|
||||
this.processResults_(rows, parsedQuery);
|
||||
if (searchType === SearchEngine.SEARCH_TYPE_FTS_FUZZY && !parsedQuery.any) {
|
||||
return rows.filter(row => row.include);
|
||||
}
|
||||
return rows;
|
||||
} catch (error) {
|
||||
this.logger().warn(`Cannot execute MATCH query: ${searchString}: ${error.message}`);
|
||||
@ -567,5 +666,6 @@ SearchEngine.instance_ = null;
|
||||
SearchEngine.SEARCH_TYPE_AUTO = 'auto';
|
||||
SearchEngine.SEARCH_TYPE_BASIC = 'basic';
|
||||
SearchEngine.SEARCH_TYPE_FTS = 'fts';
|
||||
SearchEngine.SEARCH_TYPE_FTS_FUZZY = 'fts_fuzzy';
|
||||
|
||||
module.exports = SearchEngine;
|
||||
|
@ -11,7 +11,7 @@ class SearchEngineUtils {
|
||||
searchType = SearchEngine.SEARCH_TYPE_BASIC;
|
||||
}
|
||||
|
||||
const results = await SearchEngine.instance().search(query, { searchType });
|
||||
const results = await SearchEngine.instance().search(query, { searchType, fuzzy: options.fuzzy });
|
||||
const noteIds = results.map(n => n.id);
|
||||
|
||||
// We need at least the note ID to be able to sort them below so if not
|
||||
|
@ -3,6 +3,7 @@ interface Term {
|
||||
name: string
|
||||
value: string
|
||||
negated: boolean
|
||||
quoted?: boolean
|
||||
}
|
||||
|
||||
const makeTerm = (name: string, value: string): Term => {
|
||||
@ -10,10 +11,9 @@ const makeTerm = (name: string, value: string): Term => {
|
||||
return { name: name, value: value, negated: false };
|
||||
};
|
||||
|
||||
const quoted = (s: string) => s.startsWith('"') && s.endsWith('"');
|
||||
|
||||
const quote = (s : string) => {
|
||||
const quoted = (s: string) => s.startsWith('"') && s.endsWith('"');
|
||||
|
||||
if (!quoted(s)) {
|
||||
return `"${s}"`;
|
||||
}
|
||||
@ -71,7 +71,6 @@ const parseQuery = (query: string): Term[] => {
|
||||
'altitude', 'resource', 'sourceurl']);
|
||||
|
||||
const terms = getTerms(query);
|
||||
// console.log(terms);
|
||||
|
||||
const result: Term[] = [];
|
||||
for (let i = 0; i < terms.length; i++) {
|
||||
@ -98,9 +97,9 @@ const parseQuery = (query: string): Term[] => {
|
||||
// Every word is quoted if not already.
|
||||
// By quoting the word, FTS match query will take care of removing dashes and other word seperators.
|
||||
if (value.startsWith('-')) {
|
||||
result.push({ name: 'text', value: quote(value.slice(1)) , negated: true });
|
||||
result.push({ name: 'text', value: quote(value.slice(1)) , negated: true, quoted: quoted(value) });
|
||||
} else {
|
||||
result.push({ name: 'text', value: quote(value), negated: false });
|
||||
result.push({ name: 'text', value: quote(value), negated: false, quoted: quoted(value) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -300,7 +300,7 @@ const sourceUrlFilter = (terms: Term[], conditons: string[], params: string[], r
|
||||
};
|
||||
|
||||
|
||||
const textFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation) => {
|
||||
const textFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, fuzzy: Boolean) => {
|
||||
const addExcludeTextConditions = (excludedTerms: Term[], conditions:string[], params: string[], relation: Relation) => {
|
||||
const type = excludedTerms[0].name === 'text' ? '' : `.${excludedTerms[0].name}`;
|
||||
|
||||
@ -342,7 +342,7 @@ const textFilter = (terms: Term[], conditions: string[], params: string[], relat
|
||||
if (term.name === 'text') return term.value;
|
||||
else return `${term.name}:${term.value}`;
|
||||
});
|
||||
const matchQuery = (relation === 'OR') ? termsToMatch.join(' OR ') : termsToMatch.join(' ');
|
||||
const matchQuery = (fuzzy || (relation === 'OR')) ? termsToMatch.join(' OR ') : termsToMatch.join(' ');
|
||||
params.push(matchQuery);
|
||||
}
|
||||
|
||||
@ -374,14 +374,11 @@ const getConnective = (terms: Term[], relation: Relation): string => {
|
||||
return (!notebookTerm && (relation === 'OR')) ? 'ROWID=-1' : '1'; // ROWID=-1 acts as 0 (something always false)
|
||||
};
|
||||
|
||||
export default function queryBuilder(terms: Term[]) {
|
||||
export default function queryBuilder(terms: Term[], fuzzy: boolean) {
|
||||
const queryParts: string[] = [];
|
||||
const params: string[] = [];
|
||||
const withs: string[] = [];
|
||||
|
||||
// console.log("testing beep beep boop boop")
|
||||
// console.log(terms);
|
||||
|
||||
const relation: Relation = getDefaultRelation(terms);
|
||||
|
||||
queryParts.push(`
|
||||
@ -405,8 +402,8 @@ export default function queryBuilder(terms: Term[]) {
|
||||
|
||||
resourceFilter(terms, queryParts, params, relation, withs);
|
||||
|
||||
textFilter(terms, queryParts, params, relation, fuzzy);
|
||||
|
||||
textFilter(terms, queryParts, params, relation);
|
||||
|
||||
typeFilter(terms, queryParts, params, relation);
|
||||
|
||||
|
BIN
ReactNativeClient/lib/sql-extensions/spellfix.dll
Normal file
BIN
ReactNativeClient/lib/sql-extensions/spellfix.dll
Normal file
Binary file not shown.
50
Tools/gulp/tasks/compileExtensions.js
Normal file
50
Tools/gulp/tasks/compileExtensions.js
Normal file
@ -0,0 +1,50 @@
|
||||
const fs = require('fs-extra');
|
||||
const utils = require('../utils');
|
||||
|
||||
|
||||
async function getSourceCode(dest) {
|
||||
await utils.execCommand(`curl -o ${dest}/sqlite.tar.gz "https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release"`);
|
||||
await utils.execCommand(`curl -o ${dest}/amalgamation.tar.gz "https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz"`);
|
||||
await utils.execCommand(`tar -xzvf ${dest}/sqlite.tar.gz -C ${dest}`);
|
||||
await utils.execCommand(`tar -xzvf ${dest}/amalgamation.tar.gz -C ${dest}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const rootDir = utils.rootDir();
|
||||
const dest = `${rootDir}/ReactNativeClient/lib/sql-extensions`;
|
||||
|
||||
try {
|
||||
await fs.ensureDir(dest);
|
||||
|
||||
if (utils.isLinux()) {
|
||||
try {
|
||||
await fs.promises.access(`${dest}/spellfix.so`);
|
||||
} catch (e) {
|
||||
await getSourceCode(dest);
|
||||
await utils.execCommand(`gcc -shared -fPIC -Wall -I${dest}/sqlite-autoconf-3330000/ ${dest}/sqlite/ext/misc/spellfix.c -o ${dest}/spellfix.so`);
|
||||
}
|
||||
}
|
||||
|
||||
if (utils.isMac()) {
|
||||
try {
|
||||
await fs.promises.access(`${dest}/spellfix.dylib`);
|
||||
} catch (e) {
|
||||
await getSourceCode(dest);
|
||||
await utils.execCommand(`gcc -shared -fPIC -Wall -I${dest}/sqlite-autoconf-3330000/ -dynamiclib ${dest}/sqlite/ext/misc/spellfix.c -o ${dest}/spellfix.dylib`);
|
||||
}
|
||||
}
|
||||
|
||||
// if (utils.isWindows()) {
|
||||
// try {
|
||||
// await fs.promises.access(`${dest}/spellfix.dll`);
|
||||
// } catch (e) {
|
||||
// await getSourceCode(dest);
|
||||
// await utils.execCommand(`cl /I ${dest}/sqlite-autoconf-3330000 ${dest}/sqlite/ext/misc/spellfix.c -link -dll -out:spellfix.dll `)
|
||||
// }
|
||||
// }
|
||||
} catch (e) {
|
||||
console.warn(e);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = main;
|
Loading…
Reference in New Issue
Block a user