1
0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00

Started integrating search engine to desktop app

This commit is contained in:
Laurent Cozic 2018-12-10 19:58:49 +01:00
parent 460f826672
commit 0a6f8b0cfe
5 changed files with 60 additions and 9 deletions

View File

@ -23,6 +23,7 @@ const DecryptionWorker = require('lib/services/DecryptionWorker');
const InteropService = require('lib/services/InteropService');
const InteropServiceHelper = require('./InteropServiceHelper.js');
const ResourceService = require('lib/services/ResourceService');
const SearchEngine = require('lib/services/SearchEngine');
const ClipperServer = require('lib/ClipperServer');
const ExternalEditWatcher = require('lib/services/ExternalEditWatcher');
const { bridge } = require('electron').remote.require('./bridge');
@ -793,6 +794,10 @@ class Application extends BaseApplication {
ResourceService.runInBackground();
SearchEngine.instance().setDb(reg.db());
SearchEngine.instance().setLogger(reg.logger());
SearchEngine.runInBackground();
if (Setting.value('env') === 'dev') {
AlarmService.updateAllNotifications();
} else {

View File

@ -34,6 +34,7 @@ const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js');
const SyncTargetDropbox = require('lib/SyncTargetDropbox.js');
const EncryptionService = require('lib/services/EncryptionService');
const ResourceFetcher = require('lib/services/ResourceFetcher');
const SearchEngine = require('lib/services/SearchEngine');
const DecryptionWorker = require('lib/services/DecryptionWorker');
const BaseService = require('lib/services/BaseService');
@ -218,12 +219,28 @@ class BaseApplication {
} else if (parentType === Tag.modelType()) {
notes = await Tag.notes(parentId, options);
} else if (parentType === BaseModel.TYPE_SEARCH) {
let fields = Note.previewFields();
let search = BaseModel.byId(state.searches, parentId);
notes = await Note.previews(null, {
fields: fields,
anywherePattern: '*' + search.query_pattern + '*',
});
const search = BaseModel.byId(state.searches, parentId);
const results = await SearchEngine.instance().search(search.query_pattern);
const noteIds = results.map(n => n.id);
const previewOptions = {
order: [],
fields: Note.previewFields(),
conditions: ['id IN ("' + noteIds.join('","') + '")'],
}
notes = await Note.previews(null, previewOptions);
// By default, the notes will be returned in reverse order
// or maybe random order so sort them here in the correct order
// (search engine returns the results in order of relevance).
const sortedNotes = [];
for (let i = 0; i < notes.length; i++) {
const idx = noteIds.indexOf(notes[i].id);
sortedNotes[idx] = notes[i];
}
notes = sortedNotes;
}
}

View File

@ -219,6 +219,7 @@ class JoplinDatabase extends Database {
if (tableName == 'android_metadata') continue;
if (tableName == 'table_fields') continue;
if (tableName == 'sqlite_sequence') continue;
if (tableName.indexOf('notes_fts') === 0) continue;
chain.push(() => {
return this.selectAll('PRAGMA table_info("' + tableName + '")').then((pragmas) => {
for (let i = 0; i < pragmas.length; i++) {
@ -260,7 +261,7 @@ class JoplinDatabase extends Database {
// default value and thus might cause problems. In that case, the default value
// must be set in the synchronizer too.
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14];
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
@ -445,6 +446,12 @@ class JoplinDatabase extends Database {
}));
}
if (targetVersion == 15) {
// NOTE: Duplicated from SearchEngine.createFtsTables()
queries.push('CREATE VIRTUAL TABLE notes_fts USING fts4(content="notes", notindexed="id", id, title, body)');
queries.push('INSERT INTO notes_fts(docid, id, title, body) SELECT rowid, id, title, body FROM notes WHERE is_conflict = 0 AND encryption_applied = 0');
}
queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });
await this.transactionExecBatch(queries);

View File

@ -249,7 +249,7 @@ class Note extends BaseItem {
// is used to sort already loaded notes.
if (!options) options = {};
if (!options.order) options.order = [
if (!('order' in options)) options.order = [
{ by: 'user_updated_time', dir: 'DESC' },
{ by: 'user_created_time', dir: 'DESC' },
{ by: 'title', dir: 'DESC' },

View File

@ -1,4 +1,5 @@
const { Logger } = require('lib/logger.js');
const { shim } = require('lib/shim.js');
const ItemChange = require('lib/models/ItemChange.js');
const Setting = require('lib/models/Setting.js');
const Note = require('lib/models/Note.js');
@ -12,6 +13,7 @@ class SearchEngine {
this.db_ = null;
}
// Note: Duplicated in JoplinDatabase migration 15
async createFtsTables() {
await this.db().exec('CREATE VIRTUAL TABLE notes_fts USING fts4(content="notes", notindexed="id", id, title, body)');
await this.db().exec('INSERT INTO notes_fts(docid, id, title, body) SELECT rowid, id, title, body FROM notes WHERE is_conflict = 0 AND encryption_applied = 0');
@ -27,6 +29,8 @@ class SearchEngine {
// await this.db().exec('INSERT INTO notes_fts(docid, id, title, body) SELECT rowid, id, title, body FROM notes WHERE is_conflict = 0 AND encryption_applied = 0');
// return;
this.logger().info('SearchEngine: Updating FTS table...');
await ItemChange.waitForAllSaved();
const startTime = Date.now();
@ -112,6 +116,10 @@ class SearchEngine {
calculateWeight_(offsets) {
// Offset doc: https://www.sqlite.org/fts3.html#offsets
// TODO: If there's only one term - specia case - whatever has the most occurences win.
// TODO: Parse query string.
// TODO: Support wildcards
const occurenceCount = Math.floor(offsets.length / 4);
@ -151,12 +159,26 @@ class SearchEngine {
}
async search(query) {
const sql = 'SELECT id, offsets(notes_fts) AS offsets FROM notes_fts WHERE notes_fts MATCH ?'
const sql = 'SELECT id, title, offsets(notes_fts) AS offsets FROM notes_fts WHERE notes_fts MATCH ?'
const rows = await this.db().selectAll(sql, [query]);
this.orderResults_(rows);
return rows;
}
static runInBackground() {
if (this.isRunningInBackground_) return;
this.isRunningInBackground_ = true;
setTimeout(() => {
SearchEngine.instance().updateFtsTables();
}, 1000 * 30);
shim.setInterval(() => {
SearchEngine.instance().updateFtsTables();
}, 1000 * 60 * 30);
}
}
module.exports = SearchEngine;