2021-01-22 19:41:11 +02:00
|
|
|
import Logger from '../../Logger';
|
|
|
|
import ItemChange from '../../models/ItemChange';
|
|
|
|
import Setting from '../../models/Setting';
|
|
|
|
import Note from '../../models/Note';
|
|
|
|
import BaseModel from '../../BaseModel';
|
|
|
|
import ItemChangeUtils from '../ItemChangeUtils';
|
|
|
|
import shim from '../../shim';
|
|
|
|
import filterParser from './filterParser';
|
|
|
|
import queryBuilder from './queryBuilder';
|
|
|
|
import { ItemChangeEntity, NoteEntity } from '../database/types';
|
2019-06-28 01:48:52 +02:00
|
|
|
const { sprintf } = require('sprintf-js');
|
2021-01-22 19:41:11 +02:00
|
|
|
const { pregQuote, scriptType, removeDiacritics } = require('../../string-utils.js');
|
2018-12-09 22:45:50 +02:00
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
export default class SearchEngine {
|
2020-08-08 01:13:21 +02:00
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
public static instance_: SearchEngine = null;
|
2021-04-29 16:27:38 +02:00
|
|
|
public static relevantFields = 'id, title, body, user_created_time, user_updated_time, is_todo, todo_completed, todo_due, parent_id, latitude, longitude, altitude, source_url';
|
2021-01-22 19:41:11 +02:00
|
|
|
public static SEARCH_TYPE_AUTO = 'auto';
|
|
|
|
public static SEARCH_TYPE_BASIC = 'basic';
|
2021-06-07 16:15:04 +02:00
|
|
|
public static SEARCH_TYPE_NONLATIN_SCRIPT = 'nonlatin';
|
2021-01-22 19:41:11 +02:00
|
|
|
public static SEARCH_TYPE_FTS = 'fts';
|
|
|
|
|
|
|
|
public dispatch: Function = (_o: any) => {};
|
|
|
|
private logger_ = new Logger();
|
|
|
|
private db_: any = null;
|
|
|
|
private isIndexing_ = false;
|
|
|
|
private syncCalls_: any[] = [];
|
|
|
|
private scheduleSyncTablesIID_: any;
|
2018-12-10 20:54:46 +02:00
|
|
|
|
2018-12-09 22:45:50 +02:00
|
|
|
static instance() {
|
2020-02-27 20:25:42 +02:00
|
|
|
if (SearchEngine.instance_) return SearchEngine.instance_;
|
|
|
|
SearchEngine.instance_ = new SearchEngine();
|
|
|
|
return SearchEngine.instance_;
|
2018-12-09 22:45:50 +02:00
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
setLogger(logger: Logger) {
|
2018-12-09 22:45:50 +02:00
|
|
|
this.logger_ = logger;
|
|
|
|
}
|
|
|
|
|
|
|
|
logger() {
|
|
|
|
return this.logger_;
|
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
setDb(db: any) {
|
2018-12-09 22:45:50 +02:00
|
|
|
this.db_ = db;
|
|
|
|
}
|
|
|
|
|
|
|
|
db() {
|
|
|
|
return this.db_;
|
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
noteById_(notes: NoteEntity[], noteId: string) {
|
2019-01-13 18:05:07 +02:00
|
|
|
for (let i = 0; i < notes.length; i++) {
|
|
|
|
if (notes[i].id === noteId) return notes[i];
|
|
|
|
}
|
|
|
|
// The note may have been deleted since the change was recorded. For example in this case:
|
|
|
|
// - Note created (Some Change object is recorded)
|
|
|
|
// - Note is deleted
|
|
|
|
// - ResourceService indexer runs.
|
|
|
|
// In that case, there will be a change for the note, but the note will be gone.
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
2019-01-15 20:10:22 +02:00
|
|
|
async rebuildIndex_() {
|
2021-01-22 19:41:11 +02:00
|
|
|
let noteIds: string[] = await this.db().selectAll('SELECT id FROM notes WHERE is_conflict = 0 AND encryption_applied = 0');
|
|
|
|
noteIds = noteIds.map((n: any) => n.id);
|
2019-01-13 18:05:07 +02:00
|
|
|
|
2019-01-14 21:11:54 +02:00
|
|
|
const lastChangeId = await ItemChange.lastChangeId();
|
2019-01-13 18:05:07 +02:00
|
|
|
|
|
|
|
// First delete content of note_normalized, in case the previous initial indexing failed
|
2019-01-14 21:11:54 +02:00
|
|
|
await this.db().exec('DELETE FROM notes_normalized');
|
2019-01-13 18:05:07 +02:00
|
|
|
|
|
|
|
while (noteIds.length) {
|
|
|
|
const currentIds = noteIds.splice(0, 100);
|
2020-08-08 01:13:21 +02:00
|
|
|
const notes = await Note.modelSelectAll(`
|
|
|
|
SELECT ${SearchEngine.relevantFields}
|
|
|
|
FROM notes
|
|
|
|
WHERE id IN ("${currentIds.join('","')}") AND is_conflict = 0 AND encryption_applied = 0`);
|
2019-01-13 18:05:07 +02:00
|
|
|
const queries = [];
|
|
|
|
|
|
|
|
for (let i = 0; i < notes.length; i++) {
|
|
|
|
const note = notes[i];
|
|
|
|
const n = this.normalizeNote_(note);
|
2020-08-08 01:13:21 +02:00
|
|
|
queries.push({ sql: `
|
|
|
|
INSERT INTO notes_normalized(${SearchEngine.relevantFields})
|
2021-04-29 16:27:38 +02:00
|
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
|
|
params: [n.id, n.title, n.body, n.user_created_time, n.user_updated_time, n.is_todo, n.todo_completed, n.todo_due, n.parent_id, n.latitude, n.longitude, n.altitude, n.source_url] }
|
2020-08-08 01:13:21 +02:00
|
|
|
);
|
2019-01-13 18:05:07 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
await this.db().transactionExecBatch(queries);
|
|
|
|
}
|
|
|
|
|
2019-01-14 21:11:54 +02:00
|
|
|
Setting.setValue('searchEngine.lastProcessedChangeId', lastChangeId);
|
2019-01-13 18:05:07 +02:00
|
|
|
}
|
|
|
|
|
2019-01-15 20:10:22 +02:00
|
|
|
scheduleSyncTables() {
|
|
|
|
if (this.scheduleSyncTablesIID_) return;
|
|
|
|
|
2020-10-09 19:35:46 +02:00
|
|
|
this.scheduleSyncTablesIID_ = shim.setTimeout(async () => {
|
2019-06-26 19:36:42 +02:00
|
|
|
try {
|
|
|
|
await this.syncTables();
|
|
|
|
} catch (error) {
|
|
|
|
this.logger().error('SearchEngine::scheduleSyncTables: Error while syncing tables:', error);
|
|
|
|
}
|
2019-01-15 20:10:22 +02:00
|
|
|
this.scheduleSyncTablesIID_ = null;
|
|
|
|
}, 10000);
|
|
|
|
}
|
|
|
|
|
2019-06-28 01:48:52 +02:00
|
|
|
async rebuildIndex() {
|
2019-07-29 15:43:53 +02:00
|
|
|
Setting.setValue('searchEngine.lastProcessedChangeId', 0);
|
2019-06-28 01:48:52 +02:00
|
|
|
Setting.setValue('searchEngine.initialIndexingDone', false);
|
|
|
|
return this.syncTables();
|
|
|
|
}
|
|
|
|
|
2020-02-22 13:25:16 +02:00
|
|
|
async syncTables_() {
|
2019-01-15 20:10:22 +02:00
|
|
|
if (this.isIndexing_) return;
|
|
|
|
|
|
|
|
this.isIndexing_ = true;
|
|
|
|
|
2018-12-29 21:19:18 +02:00
|
|
|
this.logger().info('SearchEngine: Updating FTS table...');
|
|
|
|
|
|
|
|
await ItemChange.waitForAllSaved();
|
|
|
|
|
2019-01-13 18:05:07 +02:00
|
|
|
if (!Setting.value('searchEngine.initialIndexingDone')) {
|
2019-01-15 20:10:22 +02:00
|
|
|
await this.rebuildIndex_();
|
|
|
|
Setting.setValue('searchEngine.initialIndexingDone', true);
|
|
|
|
this.isIndexing_ = false;
|
2019-01-13 18:05:07 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-12-29 21:19:18 +02:00
|
|
|
const startTime = Date.now();
|
|
|
|
|
2019-06-28 01:48:52 +02:00
|
|
|
const report = {
|
|
|
|
inserted: 0,
|
2019-07-29 15:43:53 +02:00
|
|
|
deleted: 0,
|
2019-06-28 01:48:52 +02:00
|
|
|
};
|
|
|
|
|
2018-12-29 21:19:18 +02:00
|
|
|
let lastChangeId = Setting.value('searchEngine.lastProcessedChangeId');
|
|
|
|
|
2019-06-26 19:36:42 +02:00
|
|
|
try {
|
|
|
|
while (true) {
|
2021-01-22 19:41:11 +02:00
|
|
|
const changes: ItemChangeEntity[] = await ItemChange.modelSelectAll(
|
2019-07-29 15:43:53 +02:00
|
|
|
`
|
2019-06-26 19:36:42 +02:00
|
|
|
SELECT id, item_id, type
|
|
|
|
FROM item_changes
|
|
|
|
WHERE item_type = ?
|
|
|
|
AND id > ?
|
|
|
|
ORDER BY id ASC
|
2019-06-29 00:49:43 +02:00
|
|
|
LIMIT 10
|
2019-07-29 15:43:53 +02:00
|
|
|
`,
|
|
|
|
[BaseModel.TYPE_NOTE, lastChangeId]
|
|
|
|
);
|
2019-06-26 19:36:42 +02:00
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
if (!changes.length) break;
|
2020-09-06 14:07:00 +02:00
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
const queries = [];
|
2019-06-26 19:36:42 +02:00
|
|
|
|
2020-05-21 10:14:33 +02:00
|
|
|
const noteIds = changes.map(a => a.item_id);
|
2020-08-08 01:13:21 +02:00
|
|
|
const notes = await Note.modelSelectAll(`
|
|
|
|
SELECT ${SearchEngine.relevantFields}
|
|
|
|
FROM notes WHERE id IN ("${noteIds.join('","')}") AND is_conflict = 0 AND encryption_applied = 0`
|
|
|
|
);
|
|
|
|
|
2019-06-26 19:36:42 +02:00
|
|
|
for (let i = 0; i < changes.length; i++) {
|
|
|
|
const change = changes[i];
|
|
|
|
|
|
|
|
if (change.type === ItemChange.TYPE_CREATE || change.type === ItemChange.TYPE_UPDATE) {
|
|
|
|
queries.push({ sql: 'DELETE FROM notes_normalized WHERE id = ?', params: [change.item_id] });
|
|
|
|
const note = this.noteById_(notes, change.item_id);
|
|
|
|
if (note) {
|
|
|
|
const n = this.normalizeNote_(note);
|
2020-08-08 01:13:21 +02:00
|
|
|
queries.push({ sql: `
|
|
|
|
INSERT INTO notes_normalized(${SearchEngine.relevantFields})
|
2021-04-29 16:27:38 +02:00
|
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
|
|
params: [change.item_id, n.title, n.body, n.user_created_time, n.user_updated_time, n.is_todo, n.todo_completed, n.todo_due, n.parent_id, n.latitude, n.longitude, n.altitude, n.source_url] });
|
2019-06-28 01:48:52 +02:00
|
|
|
report.inserted++;
|
2019-06-26 19:36:42 +02:00
|
|
|
}
|
|
|
|
} else if (change.type === ItemChange.TYPE_DELETE) {
|
|
|
|
queries.push({ sql: 'DELETE FROM notes_normalized WHERE id = ?', params: [change.item_id] });
|
2019-06-28 01:48:52 +02:00
|
|
|
report.deleted++;
|
2019-06-26 19:36:42 +02:00
|
|
|
} else {
|
2019-09-19 23:51:18 +02:00
|
|
|
throw new Error(`Invalid change type: ${change.type}`);
|
2019-01-13 18:05:07 +02:00
|
|
|
}
|
2019-06-26 19:36:42 +02:00
|
|
|
|
|
|
|
lastChangeId = change.id;
|
2018-12-29 21:19:18 +02:00
|
|
|
}
|
|
|
|
|
2019-06-26 19:36:42 +02:00
|
|
|
await this.db().transactionExecBatch(queries);
|
|
|
|
Setting.setValue('searchEngine.lastProcessedChangeId', lastChangeId);
|
|
|
|
await Setting.saveAll();
|
2018-12-29 21:19:18 +02:00
|
|
|
}
|
2019-06-26 19:36:42 +02:00
|
|
|
} catch (error) {
|
|
|
|
this.logger().error('SearchEngine: Error while processing changes:', error);
|
2018-12-29 21:19:18 +02:00
|
|
|
}
|
|
|
|
|
2019-01-14 21:11:54 +02:00
|
|
|
await ItemChangeUtils.deleteProcessedChanges();
|
|
|
|
|
2019-06-28 01:48:52 +02:00
|
|
|
this.logger().info(sprintf('SearchEngine: Updated FTS table in %dms. Inserted: %d. Deleted: %d', Date.now() - startTime, report.inserted, report.deleted));
|
2019-01-15 20:10:22 +02:00
|
|
|
|
|
|
|
this.isIndexing_ = false;
|
2019-02-09 21:04:34 +02:00
|
|
|
}
|
2018-12-29 21:19:18 +02:00
|
|
|
|
2020-02-22 13:25:16 +02:00
|
|
|
async syncTables() {
|
|
|
|
this.syncCalls_.push(true);
|
|
|
|
try {
|
|
|
|
await this.syncTables_();
|
|
|
|
} finally {
|
|
|
|
this.syncCalls_.pop();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-12-10 20:54:46 +02:00
|
|
|
async countRows() {
|
2019-07-29 15:43:53 +02:00
|
|
|
const sql = 'SELECT count(*) as total FROM notes_fts';
|
2018-12-10 20:54:46 +02:00
|
|
|
const row = await this.db().selectOne(sql);
|
|
|
|
return row && row['total'] ? row['total'] : 0;
|
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
fieldNamesFromOffsets_(offsets: any[]) {
|
2020-04-14 00:10:59 +02:00
|
|
|
const notesNormalizedFieldNames = this.db().tableFieldNames('notes_normalized');
|
2018-12-10 20:54:46 +02:00
|
|
|
const occurenceCount = Math.floor(offsets.length / 4);
|
2021-01-22 19:41:11 +02:00
|
|
|
const output: string[] = [];
|
2018-12-10 20:54:46 +02:00
|
|
|
for (let i = 0; i < occurenceCount; i++) {
|
2020-04-14 00:10:59 +02:00
|
|
|
const colIndex = offsets[i * 4];
|
|
|
|
const fieldName = notesNormalizedFieldNames[colIndex];
|
|
|
|
if (!output.includes(fieldName)) output.push(fieldName);
|
2018-12-10 20:54:46 +02:00
|
|
|
}
|
|
|
|
|
2020-04-14 00:10:59 +02:00
|
|
|
return output;
|
2018-12-10 20:54:46 +02:00
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
calculateWeight_(offsets: any[], termCount: number) {
|
2018-12-10 20:54:46 +02:00
|
|
|
// Offset doc: https://www.sqlite.org/fts3.html#offsets
|
2018-12-10 20:58:49 +02:00
|
|
|
|
2018-12-12 23:40:05 +02:00
|
|
|
// - If there's only one term in the query string, the content with the most matches goes on top
|
|
|
|
// - If there are multiple terms, the result with the most occurences that are closest to each others go on top.
|
|
|
|
// eg. if query is "abcd efgh", "abcd efgh" will go before "abcd XX efgh".
|
2019-07-29 15:43:53 +02:00
|
|
|
|
2018-12-10 20:54:46 +02:00
|
|
|
const occurenceCount = Math.floor(offsets.length / 4);
|
|
|
|
|
2018-12-12 23:40:05 +02:00
|
|
|
if (termCount === 1) return occurenceCount;
|
|
|
|
|
2018-12-10 20:54:46 +02:00
|
|
|
let spread = 0;
|
|
|
|
let previousDist = null;
|
|
|
|
for (let i = 0; i < occurenceCount; i++) {
|
|
|
|
const dist = offsets[i * 4 + 2];
|
|
|
|
|
|
|
|
if (previousDist !== null) {
|
|
|
|
const delta = dist - previousDist;
|
|
|
|
spread += delta;
|
|
|
|
}
|
|
|
|
|
|
|
|
previousDist = dist;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Divide the number of occurences by the spread so even if a note has many times the searched terms
|
|
|
|
// but these terms are very spread appart, they'll be given a lower weight than a note that has the
|
|
|
|
// terms once or twice but just next to each others.
|
|
|
|
return occurenceCount / spread;
|
|
|
|
}
|
|
|
|
|
2020-08-19 00:53:28 +02:00
|
|
|
|
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
calculateWeightBM25_(rows: any[]) {
|
2020-08-19 00:53:28 +02:00
|
|
|
// https://www.sqlite.org/fts3.html#matchinfo
|
|
|
|
// pcnalx are the arguments passed to matchinfo
|
|
|
|
// p - The number of matchable phrases in the query.
|
|
|
|
// c - The number of user defined columns in the FTS table
|
|
|
|
// n - The number of rows in the FTS4 table.
|
|
|
|
// a - avg number of tokens in the text values stored in the column.
|
|
|
|
// l - For each column, the length of the value stored in the current
|
|
|
|
// row of the FTS4 table, in tokens.
|
|
|
|
// x - For each distinct combination of a phrase and table column, the
|
|
|
|
// following three values:
|
|
|
|
// hits_this_row
|
|
|
|
// hits_all_rows
|
|
|
|
// docs_with_hits
|
|
|
|
|
|
|
|
if (rows.length === 0) return;
|
|
|
|
|
|
|
|
const matchInfo = rows.map(row => new Uint32Array(row.matchinfo.buffer));
|
|
|
|
const generalInfo = matchInfo[0];
|
|
|
|
|
|
|
|
const K1 = 1.2;
|
|
|
|
const B = 0.75;
|
|
|
|
|
|
|
|
const TITLE_COLUMN = 1;
|
|
|
|
const BODY_COLUMN = 2;
|
|
|
|
const columns = [TITLE_COLUMN, BODY_COLUMN];
|
|
|
|
// const NUM_COLS = 12;
|
|
|
|
|
|
|
|
const numPhrases = generalInfo[0]; // p
|
|
|
|
const numColumns = generalInfo[1]; // c
|
|
|
|
const numRows = generalInfo[2]; // n
|
|
|
|
|
|
|
|
const avgTitleTokens = generalInfo[4]; // a
|
|
|
|
const avgBodyTokens = generalInfo[5];
|
|
|
|
const avgTokens = [null, avgTitleTokens, avgBodyTokens]; // we only need cols 1 and 2
|
|
|
|
|
|
|
|
const numTitleTokens = matchInfo.map(m => m[4 + numColumns]); // l
|
|
|
|
const numBodyTokens = matchInfo.map(m => m[5 + numColumns]);
|
|
|
|
const numTokens = [null, numTitleTokens, numBodyTokens];
|
|
|
|
|
|
|
|
const X = matchInfo.map(m => m.slice(27)); // x
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
const hitsThisRow = (array: any, c: number, p: number) => array[3 * (c + p * numColumns) + 0];
|
2020-08-19 00:53:28 +02:00
|
|
|
// const hitsAllRows = (array, c, p) => array[3 * (c + p*NUM_COLS) + 1];
|
2021-01-22 19:41:11 +02:00
|
|
|
const docsWithHits = (array: any, c: number, p: number) => array[3 * (c + p * numColumns) + 2];
|
2020-08-19 00:53:28 +02:00
|
|
|
|
|
|
|
|
|
|
|
// if a term occurs in over half the documents in the collection
|
|
|
|
// then this model gives a negative term weight, which is presumably undesirable.
|
|
|
|
// But, assuming the use of a stop list, this normally doesn't happen,
|
|
|
|
// and the value for each summand can be given a floor of 0.
|
2021-01-22 19:41:11 +02:00
|
|
|
const IDF = (n: number, N: number) => Math.max(Math.log((N - n + 0.5) / (n + 0.5)), 0);
|
2020-08-19 00:53:28 +02:00
|
|
|
|
|
|
|
// https://en.wikipedia.org/wiki/Okapi_BM25
|
2021-01-22 19:41:11 +02:00
|
|
|
const BM25 = (idf: any, freq: any, numTokens: number, avgTokens: any) => {
|
2020-08-19 00:53:28 +02:00
|
|
|
if (avgTokens === 0) {
|
|
|
|
return 0; // To prevent division by zero
|
|
|
|
}
|
|
|
|
return idf * (freq * (K1 + 1)) / (freq + K1 * (1 - B + B * (numTokens / avgTokens)));
|
|
|
|
};
|
|
|
|
|
2020-10-09 22:51:11 +02:00
|
|
|
const msSinceEpoch = Math.round(new Date().getTime());
|
|
|
|
const msPerDay = 86400000;
|
2021-01-22 19:41:11 +02:00
|
|
|
const weightForDaysSinceLastUpdate = (row: any) => {
|
2020-10-09 22:51:11 +02:00
|
|
|
// BM25 weights typically range 0-10, and last updated date should weight similarly, though prioritizing recency logarithmically.
|
|
|
|
// An alpha of 200 ensures matches in the last week will show up front (11.59) and often so for matches within 2 weeks (5.99),
|
|
|
|
// but is much less of a factor at 30 days (2.84) or very little after 90 days (0.95), focusing mostly on content at that point.
|
|
|
|
if (!row.user_updated_time) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
const alpha = 200;
|
|
|
|
const daysSinceLastUpdate = (msSinceEpoch - row.user_updated_time) / msPerDay;
|
|
|
|
return alpha * Math.log(1 + 1 / Math.max(daysSinceLastUpdate, 0.5));
|
|
|
|
};
|
|
|
|
|
2020-08-19 00:53:28 +02:00
|
|
|
for (let i = 0; i < rows.length; i++) {
|
|
|
|
const row = rows[i];
|
|
|
|
row.weight = 0;
|
|
|
|
for (let j = 0; j < numPhrases; j++) {
|
|
|
|
columns.forEach(column => {
|
|
|
|
const rowsWithHits = docsWithHits(X[i], column, j);
|
|
|
|
const frequencyHits = hitsThisRow(X[i], column, j);
|
|
|
|
const idf = IDF(rowsWithHits, numRows);
|
2020-09-06 14:07:00 +02:00
|
|
|
|
2020-08-19 00:53:28 +02:00
|
|
|
row.weight += BM25(idf, frequencyHits, numTokens[column][i], avgTokens[column]);
|
|
|
|
});
|
|
|
|
}
|
2020-10-09 22:51:11 +02:00
|
|
|
|
|
|
|
row.weight += weightForDaysSinceLastUpdate(row);
|
2020-08-19 00:53:28 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
processBasicSearchResults_(rows: any[], parsedQuery: any) {
|
|
|
|
const valueRegexs = parsedQuery.keys.includes('_') ? parsedQuery.terms['_'].map((term: any) => term.valueRegex || term.value) : [];
|
2020-06-03 18:06:14 +02:00
|
|
|
const isTitleSearch = parsedQuery.keys.includes('title');
|
|
|
|
const isOnlyTitle = parsedQuery.keys.length === 1 && isTitleSearch;
|
|
|
|
|
2018-12-10 20:54:46 +02:00
|
|
|
for (let i = 0; i < rows.length; i++) {
|
|
|
|
const row = rows[i];
|
2021-01-22 19:41:11 +02:00
|
|
|
const testTitle = (regex: any) => new RegExp(regex, 'ig').test(row.title);
|
|
|
|
const matchedFields: any = {
|
2020-06-03 18:06:14 +02:00
|
|
|
title: isTitleSearch || valueRegexs.some(testTitle),
|
|
|
|
body: !isOnlyTitle,
|
|
|
|
};
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
row.fields = Object.keys(matchedFields).filter((key: any) => matchedFields[key]);
|
2020-06-03 18:06:14 +02:00
|
|
|
row.weight = 0;
|
2020-09-06 14:07:00 +02:00
|
|
|
row.fuzziness = 0;
|
2020-06-03 18:06:14 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
processResults_(rows: any[], parsedQuery: any, isBasicSearchResults = false) {
|
2020-06-03 18:06:14 +02:00
|
|
|
if (isBasicSearchResults) {
|
|
|
|
this.processBasicSearchResults_(rows, parsedQuery);
|
|
|
|
} else {
|
2021-03-11 00:27:45 +02:00
|
|
|
this.calculateWeightBM25_(rows);
|
2020-06-03 18:06:14 +02:00
|
|
|
for (let i = 0; i < rows.length; i++) {
|
|
|
|
const row = rows[i];
|
2021-01-22 19:41:11 +02:00
|
|
|
const offsets = row.offsets.split(' ').map((o: any) => Number(o));
|
2020-06-03 18:06:14 +02:00
|
|
|
row.fields = this.fieldNamesFromOffsets_(offsets);
|
|
|
|
}
|
2018-12-10 20:54:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
rows.sort((a, b) => {
|
2020-04-14 00:10:59 +02:00
|
|
|
if (a.fields.includes('title') && !b.fields.includes('title')) return -1;
|
|
|
|
if (!a.fields.includes('title') && b.fields.includes('title')) return +1;
|
2018-12-10 20:54:46 +02:00
|
|
|
if (a.weight < b.weight) return +1;
|
|
|
|
if (a.weight > b.weight) return -1;
|
2019-02-24 14:00:06 +02:00
|
|
|
if (a.is_todo && a.todo_completed) return +1;
|
|
|
|
if (b.is_todo && b.todo_completed) return -1;
|
|
|
|
if (a.user_updated_time < b.user_updated_time) return +1;
|
|
|
|
if (a.user_updated_time > b.user_updated_time) return -1;
|
2018-12-10 20:54:46 +02:00
|
|
|
return 0;
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2018-12-12 23:40:05 +02:00
|
|
|
// https://stackoverflow.com/a/13818704/561309
|
2021-01-22 19:41:11 +02:00
|
|
|
queryTermToRegex(term: any) {
|
2018-12-14 00:57:14 +02:00
|
|
|
while (term.length && term.indexOf('*') === 0) {
|
|
|
|
term = term.substr(1);
|
|
|
|
}
|
|
|
|
|
2018-12-16 19:32:42 +02:00
|
|
|
let regexString = pregQuote(term);
|
2018-12-14 00:57:14 +02:00
|
|
|
if (regexString[regexString.length - 1] === '*') {
|
2019-09-19 23:51:18 +02:00
|
|
|
regexString = `${regexString.substr(0, regexString.length - 2)}[^${pregQuote(' \t\n\r,.,+-*?!={}<>|:"\'()[]')}]` + '*?';
|
2019-01-17 21:01:35 +02:00
|
|
|
// regexString = regexString.substr(0, regexString.length - 2) + '.*?';
|
2018-12-12 23:40:05 +02:00
|
|
|
}
|
2018-12-14 00:57:14 +02:00
|
|
|
|
|
|
|
return regexString;
|
2018-12-10 20:54:46 +02:00
|
|
|
}
|
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
async parseQuery(query: string) {
|
2020-09-15 15:01:07 +02:00
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
const trimQuotes = (str: string) => str.startsWith('"') ? str.substr(1, str.length - 2) : str;
|
2018-12-10 20:58:49 +02:00
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
let allTerms: any[] = [];
|
2020-09-06 14:07:00 +02:00
|
|
|
|
2020-08-08 01:13:21 +02:00
|
|
|
try {
|
|
|
|
allTerms = filterParser(query);
|
|
|
|
} catch (error) {
|
|
|
|
console.warn(error);
|
2018-12-12 23:40:05 +02:00
|
|
|
}
|
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
const textTerms = allTerms.filter(x => x.name === 'text' && !x.negated).map(x => trimQuotes(x.value));
|
|
|
|
const titleTerms = allTerms.filter(x => x.name === 'title' && !x.negated).map(x => trimQuotes(x.value));
|
|
|
|
const bodyTerms = allTerms.filter(x => x.name === 'body' && !x.negated).map(x => trimQuotes(x.value));
|
2020-09-28 19:58:19 +02:00
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
const terms: any = { _: textTerms, 'title': titleTerms, 'body': bodyTerms };
|
2018-12-12 23:40:05 +02:00
|
|
|
|
|
|
|
// Filter terms:
|
|
|
|
// - Convert wildcards to regex
|
|
|
|
// - Remove columns with no results
|
|
|
|
// - Add count of terms
|
|
|
|
|
|
|
|
let termCount = 0;
|
|
|
|
const keys = [];
|
2020-03-14 01:46:14 +02:00
|
|
|
for (const col in terms) {
|
2018-12-12 23:40:05 +02:00
|
|
|
if (!terms.hasOwnProperty(col)) continue;
|
|
|
|
|
|
|
|
if (!terms[col].length) {
|
|
|
|
delete terms[col];
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (let i = terms[col].length - 1; i >= 0; i--) {
|
|
|
|
const term = terms[col][i];
|
|
|
|
|
|
|
|
// SQlLite FTS doesn't allow "*" queries and neither shall we
|
|
|
|
if (term === '*') {
|
|
|
|
terms[col].splice(i, 1);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (term.indexOf('*') >= 0) {
|
2019-01-18 20:31:07 +02:00
|
|
|
terms[col][i] = { type: 'regex', value: term, scriptType: scriptType(term), valueRegex: this.queryTermToRegex(term) };
|
2019-01-15 21:55:58 +02:00
|
|
|
} else {
|
2019-01-18 20:31:07 +02:00
|
|
|
terms[col][i] = { type: 'text', value: term, scriptType: scriptType(term) };
|
2018-12-12 23:40:05 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
termCount += terms[col].length;
|
|
|
|
|
|
|
|
keys.push(col);
|
|
|
|
}
|
|
|
|
|
2020-10-22 13:16:47 +02:00
|
|
|
//
|
|
|
|
// The object "allTerms" is used for query construction purposes (this contains all the filter terms)
|
|
|
|
// Since this is used for the FTS match query, we need to normalize text, title and body terms.
|
|
|
|
// Note, we're not normalizing terms like tag because these are matched using SQL LIKE operator and so we must preserve their diacritics.
|
|
|
|
//
|
|
|
|
// The object "terms" only include text, title, body terms and is used for highlighting.
|
|
|
|
// By not normalizing the text, title, body in "terms", highlighting still works correctly for words with diacritics.
|
|
|
|
//
|
|
|
|
|
|
|
|
allTerms = allTerms.map(x => {
|
|
|
|
if (x.name === 'text' || x.name === 'title' || x.name === 'body') {
|
|
|
|
return Object.assign(x, { value: this.normalizeText_(x.value) });
|
|
|
|
}
|
|
|
|
return x;
|
|
|
|
});
|
|
|
|
|
2018-12-12 23:40:05 +02:00
|
|
|
return {
|
|
|
|
termCount: termCount,
|
|
|
|
keys: keys,
|
2020-08-08 01:13:21 +02:00
|
|
|
terms: terms, // text terms
|
2021-03-11 00:27:45 +02:00
|
|
|
allTerms: allTerms,
|
2020-09-06 14:07:00 +02:00
|
|
|
any: !!allTerms.find(term => term.name === 'any'),
|
2018-12-12 23:40:05 +02:00
|
|
|
};
|
2018-12-10 20:58:49 +02:00
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
allParsedQueryTerms(parsedQuery: any) {
|
2018-12-14 00:57:14 +02:00
|
|
|
if (!parsedQuery || !parsedQuery.termCount) return [];
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
let output: any[] = [];
|
2020-03-14 01:46:14 +02:00
|
|
|
for (const col in parsedQuery.terms) {
|
2018-12-14 00:57:14 +02:00
|
|
|
if (!parsedQuery.terms.hasOwnProperty(col)) continue;
|
|
|
|
output = output.concat(parsedQuery.terms[col]);
|
|
|
|
}
|
|
|
|
return output;
|
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
normalizeText_(text: string) {
|
2019-01-19 20:03:05 +02:00
|
|
|
const normalizedText = text.normalize ? text.normalize() : text;
|
|
|
|
return removeDiacritics(normalizedText.toLowerCase());
|
2019-01-13 18:05:07 +02:00
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
normalizeNote_(note: NoteEntity) {
|
2019-01-13 18:05:07 +02:00
|
|
|
const n = Object.assign({}, note);
|
2019-07-29 15:43:53 +02:00
|
|
|
n.title = this.normalizeText_(n.title);
|
2019-01-13 18:05:07 +02:00
|
|
|
n.body = this.normalizeText_(n.body);
|
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
async basicSearch(query: string) {
|
2019-04-03 08:46:41 +02:00
|
|
|
query = query.replace(/\*/, '');
|
2020-09-06 14:07:00 +02:00
|
|
|
const parsedQuery = await this.parseQuery(query);
|
2021-01-22 19:41:11 +02:00
|
|
|
const searchOptions: any = {};
|
2019-04-03 08:46:41 +02:00
|
|
|
|
|
|
|
for (const key of parsedQuery.keys) {
|
2020-06-03 18:06:14 +02:00
|
|
|
if (parsedQuery.terms[key].length === 0) continue;
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
const term = parsedQuery.terms[key].map((x: any) => x.value).join(' ');
|
2019-09-19 23:51:18 +02:00
|
|
|
if (key === '_') searchOptions.anywherePattern = `*${term}*`;
|
|
|
|
if (key === 'title') searchOptions.titlePattern = `*${term}*`;
|
|
|
|
if (key === 'body') searchOptions.bodyPattern = `*${term}*`;
|
2019-01-14 21:11:54 +02:00
|
|
|
}
|
|
|
|
|
2019-04-03 08:46:41 +02:00
|
|
|
return Note.previews(null, searchOptions);
|
2019-01-14 21:11:54 +02:00
|
|
|
}
|
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
determineSearchType_(query: string, preferredSearchType: any) {
|
|
|
|
if (preferredSearchType === SearchEngine.SEARCH_TYPE_BASIC) return SearchEngine.SEARCH_TYPE_BASIC;
|
2021-06-07 16:15:04 +02:00
|
|
|
if (preferredSearchType === SearchEngine.SEARCH_TYPE_NONLATIN_SCRIPT) return SearchEngine.SEARCH_TYPE_NONLATIN_SCRIPT;
|
2020-04-18 13:45:54 +02:00
|
|
|
|
|
|
|
// If preferredSearchType is "fts" we auto-detect anyway
|
|
|
|
// because it's not always supported.
|
2019-01-14 21:11:54 +02:00
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
let allTerms: any[] = [];
|
2020-10-22 13:16:47 +02:00
|
|
|
try {
|
|
|
|
allTerms = filterParser(query);
|
|
|
|
} catch (error) {
|
|
|
|
console.warn(error);
|
|
|
|
}
|
|
|
|
|
|
|
|
const textQuery = allTerms.filter(x => x.name === 'text' || x.name == 'title' || x.name == 'body').map(x => x.value).join(' ');
|
|
|
|
const st = scriptType(textQuery);
|
2019-01-14 21:11:54 +02:00
|
|
|
|
2021-06-07 16:15:04 +02:00
|
|
|
if (!Setting.value('db.ftsEnabled')) {
|
2020-04-18 13:45:54 +02:00
|
|
|
return SearchEngine.SEARCH_TYPE_BASIC;
|
|
|
|
}
|
|
|
|
|
2021-06-07 16:15:04 +02:00
|
|
|
// Non-alphabetical languages aren't support by SQLite FTS (except with extensions which are not available in all platforms)
|
|
|
|
if (['ja', 'zh', 'ko', 'th'].indexOf(st) >= 0) {
|
|
|
|
return SearchEngine.SEARCH_TYPE_NONLATIN_SCRIPT;
|
|
|
|
}
|
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
return SearchEngine.SEARCH_TYPE_FTS;
|
2020-04-18 13:45:54 +02:00
|
|
|
}
|
|
|
|
|
2021-01-22 19:41:11 +02:00
|
|
|
async search(searchString: string, options: any = null) {
|
2020-11-09 14:07:37 +02:00
|
|
|
if (!searchString) return [];
|
|
|
|
|
2020-04-18 13:45:54 +02:00
|
|
|
options = Object.assign({}, {
|
|
|
|
searchType: SearchEngine.SEARCH_TYPE_AUTO,
|
|
|
|
}, options);
|
|
|
|
|
2021-03-11 00:27:45 +02:00
|
|
|
const searchType = this.determineSearchType_(searchString, options.searchType);
|
|
|
|
const parsedQuery = await this.parseQuery(searchString);
|
2020-04-18 13:45:54 +02:00
|
|
|
|
|
|
|
if (searchType === SearchEngine.SEARCH_TYPE_BASIC) {
|
2020-10-22 13:16:47 +02:00
|
|
|
searchString = this.normalizeText_(searchString);
|
2020-08-08 01:13:21 +02:00
|
|
|
const rows = await this.basicSearch(searchString);
|
2021-03-11 00:27:45 +02:00
|
|
|
|
2020-06-03 18:06:14 +02:00
|
|
|
this.processResults_(rows, parsedQuery, true);
|
|
|
|
return rows;
|
2020-08-08 01:13:21 +02:00
|
|
|
} else {
|
2021-03-11 00:27:45 +02:00
|
|
|
// SEARCH_TYPE_FTS
|
2020-04-18 13:45:54 +02:00
|
|
|
// FTS will ignore all special characters, like "-" in the index. So if
|
|
|
|
// we search for "this-phrase" it won't find it because it will only
|
|
|
|
// see "this phrase" in the index. Because of this, we remove the dashes
|
|
|
|
// when searching.
|
|
|
|
// https://github.com/laurent22/joplin/issues/1075#issuecomment-459258856
|
2020-08-08 01:13:21 +02:00
|
|
|
|
2021-06-07 16:15:04 +02:00
|
|
|
const useFts = searchType === SearchEngine.SEARCH_TYPE_FTS;
|
2019-01-18 19:56:56 +02:00
|
|
|
try {
|
2021-06-07 16:15:04 +02:00
|
|
|
const { query, params } = queryBuilder(parsedQuery.allTerms, useFts);
|
2020-08-08 01:13:21 +02:00
|
|
|
const rows = await this.db().selectAll(query, params);
|
2021-06-07 16:15:04 +02:00
|
|
|
this.processResults_(rows, parsedQuery, !useFts);
|
2019-01-18 19:56:56 +02:00
|
|
|
return rows;
|
|
|
|
} catch (error) {
|
2020-08-08 01:13:21 +02:00
|
|
|
this.logger().warn(`Cannot execute MATCH query: ${searchString}: ${error.message}`);
|
2019-01-18 19:56:56 +02:00
|
|
|
return [];
|
|
|
|
}
|
2019-01-14 21:11:54 +02:00
|
|
|
}
|
|
|
|
}
|
2020-02-22 13:25:16 +02:00
|
|
|
|
2020-02-27 20:25:42 +02:00
|
|
|
async destroy() {
|
|
|
|
if (this.scheduleSyncTablesIID_) {
|
2020-10-09 19:35:46 +02:00
|
|
|
shim.clearTimeout(this.scheduleSyncTablesIID_);
|
2020-02-27 20:25:42 +02:00
|
|
|
this.scheduleSyncTablesIID_ = null;
|
|
|
|
}
|
|
|
|
SearchEngine.instance_ = null;
|
|
|
|
|
2020-02-22 13:25:16 +02:00
|
|
|
return new Promise((resolve) => {
|
2020-10-09 19:35:46 +02:00
|
|
|
const iid = shim.setInterval(() => {
|
2020-02-22 13:25:16 +02:00
|
|
|
if (!this.syncCalls_.length) {
|
2020-10-09 19:35:46 +02:00
|
|
|
shim.clearInterval(iid);
|
2021-01-22 19:41:11 +02:00
|
|
|
SearchEngine.instance_ = null;
|
|
|
|
resolve(null);
|
2020-02-22 13:25:16 +02:00
|
|
|
}
|
|
|
|
}, 100);
|
|
|
|
});
|
|
|
|
}
|
2018-12-09 22:45:50 +02:00
|
|
|
}
|