All: Add "id" and "due" search filters (#4898)

README.md

@@ -428,12 +428,13 @@ You can also use search filters to further restrict the search.
 | **title:** <br> **body:**|Restrict your search to just the title or the body field.|`title:"hello world"` searches for notes whose title contains `hello` and `world`.<br>`title:hello -body:world` searches for notes whose title contains `hello` and body does not contain `world`.
 | **tag:** |Restrict the search to the notes with the specified tags.|`tag:office` searches for all notes having tag office.<br>`tag:office tag:important` searches for all notes having both office and important tags.<br>`tag:office -tag:spam` searches for notes having tag `office` which do not have tag `spam`.<br>`any:1 tag:office tag:spam` searches for notes having tag `office` or tag `spam`.<br>`tag:be*ful` does a search with wildcards.<br>`tag:*` returns all notes with tags.<br>`-tag:*` returns all notes without tags.|
 | **notebook:** | Restrict the search to the specified notebook(s). |`notebook:books` limits the search scope within `books` and all its subnotebooks.<br>`notebook:wheel*time` does a wildcard search.|
-| **created:** <br> **updated:** | Searches for notes created/updated on dates specified using YYYYMMDD format. You can also search relative to the current day, week, month, or year. | `created:20201218` will return notes created on or after December 18, 2020.<br>`-updated:20201218` will return notes updated before December 18, 2020.<br>`created:20200118 -created:20201215` will return notes created between January 18, 2020, and before December 15, 2020.<br>`created:202001 -created:202003` will return notes created on or after January and before March 2020.<br>`updated:1997 -updated:2020` will return all notes updated between the years 1997 and 2019.<br>`created:day-2` searches for all notes created in the past two days.<br>`updated:year-0` searches all notes updated in the current year.
+| **created:** <br> **updated:** <br> **due:**| Searches for notes created/updated on dates specified using YYYYMMDD format. You can also search relative to the current day, week, month, or year. | `created:20201218` will return notes created on or after December 18, 2020.<br>`-updated:20201218` will return notes updated before December 18, 2020.<br>`created:20200118 -created:20201215` will return notes created between January 18, 2020, and before December 15, 2020.<br>`created:202001 -created:202003` will return notes created on or after January and before March 2020.<br>`updated:1997 -updated:2020` will return all notes updated between the years 1997 and 2019.<br>`created:day-2` searches for all notes created in the past two days.<br>`updated:year-0` searches all notes updated in the current year.<br>`-due:day+7` will return all todos which are due or will be due in the next seven days.<br>`-due:day-5` searches all todos that are overdue for more than 5 days.|
 | **type:** |Restrict the search to either notes or todos. | `type:note` to return all notes<br>`type:todo` to return all todos |
 | **iscompleted:** | Restrict the search to either completed or uncompleted todos. | `iscompleted:1` to return all completed todos<br>`iscompleted:0` to return all uncompleted todos|
 |**latitude:** <br> **longitude:** <br> **altitude:**|Filter by location|`latitude:40 -latitude:50` to return notes with latitude >= 40 and < 50  |
 |**resource:**|Filter by attachment MIME type|`resource:image/jpeg` to return notes with a jpeg attachment.<br>`-resource:application/pdf` to return notes without a pdf attachment.<br>`resource:image/*` to return notes with any images.|
 |**sourceurl:**|Filter by source URL|`sourceurl:https://www.google.com`<br>`sourceurl:*joplinapp.org` to perform a wildcard search.|
+|**id:**|Filter by note ID|`id:9cbc1b4f242043a9b8a50627508bccd5` return a note with the specified id |
 
 Note: In the CLI client you have to escape the query using `--` when using negated filters.
 Eg. `:search -- "-tag:tag1"`.

packages/app-cli/tests/services_SearchFilter.js

@@ -580,6 +580,64 @@ describe('services_SearchFilter', function() {
 		expect(ids(rows)).toContain(t3.id);
 	}));
 
+	it('should support filtering by due date', (async () => {
+		let rows;
+		const toDo1 = await Note.save({ title: 'ToDo 1', body: 'todo', is_todo: 1, todo_due: Date.parse('2021-04-27') });
+		const toDo2 = await Note.save({ title: 'ToDo 2', body: 'todo', is_todo: 1, todo_due: Date.parse('2021-03-17') });
+		const note1 = await Note.save({ title: 'Note 1', body: 'Note' });
+
+		await engine.syncTables();
+
+		rows = await engine.search('due:20210425');
+		expect(rows.length).toBe(1);
+		expect(ids(rows)).toContain(toDo1.id);
+
+		rows = await engine.search('-due:20210425');
+		expect(rows.length).toBe(1);
+		expect(ids(rows)).toContain(toDo2.id);
+	}));
+
+	it('should support filtering by due with smart value: day', (async () => {
+		let rows;
+
+		const inThreeDays = parseInt(time.goForwardInTime(Date.now(), 3, 'day'), 10);
+		const inSevenDays = parseInt(time.goForwardInTime(Date.now(), 7, 'day'), 10);
+		const threeDaysAgo = parseInt(time.goBackInTime(Date.now(), 3, 'day'), 10);
+		const sevenDaysAgo = parseInt(time.goBackInTime(Date.now(), 7, 'day'), 10);
+
+		const toDo1 = await Note.save({ title: 'ToDo + 3 day', body: 'toto', is_todo: 1, todo_due: inThreeDays });
+		const toDo2 = await Note.save({ title: 'ToDo + 7 day', body: 'toto', is_todo: 1, todo_due: inSevenDays });
+		const toDo3 = await Note.save({ title: 'ToDo - 3 day', body: 'toto', is_todo: 1, todo_due: threeDaysAgo });
+		const toDo4 = await Note.save({ title: 'ToDo - 7 day', body: 'toto', is_todo: 1, todo_due: sevenDaysAgo });
+
+		await engine.syncTables();
+
+		rows = await engine.search('due:day-4');
+		expect(rows.length).toBe(3);
+		expect(ids(rows)).toContain(toDo1.id);
+		expect(ids(rows)).toContain(toDo2.id);
+		expect(ids(rows)).toContain(toDo3.id);
+
+		rows = await engine.search('-due:day-4');
+		expect(rows.length).toBe(1);
+		expect(ids(rows)).toContain(toDo4.id);
+
+		rows = await engine.search('-due:day+4');
+		expect(rows.length).toBe(3);
+		expect(ids(rows)).toContain(toDo1.id);
+		expect(ids(rows)).toContain(toDo3.id);
+		expect(ids(rows)).toContain(toDo4.id);
+
+		rows = await engine.search('due:day+4');
+		expect(rows.length).toBe(1);
+		expect(ids(rows)).toContain(toDo2.id);
+
+		rows = await engine.search('due:day-4 -due:day+4');
+		expect(rows.length).toBe(2);
+		expect(ids(rows)).toContain(toDo1.id);
+		expect(ids(rows)).toContain(toDo3.id);
+	}));
+
 	it('should support filtering by latitude, longitude, altitude', (async () => {
 		let rows;
 		const n1 = await Note.save({ title: 'I made this', body: 'this week', latitude: 12.97, longitude: 88.88, altitude: 69.96 });
@@ -790,4 +848,29 @@ describe('services_SearchFilter', function() {
 
 	}));
 
+	it('should support filtering by note id', (async () => {
+		let rows;
+		const note1 = await Note.save({ title: 'Note 1', body: 'body' });
+		const note2 = await Note.save({ title: 'Note 2', body: 'body' });
+		const note3 = await Note.save({ title: 'Note 3', body: 'body' });
+		await engine.syncTables();
+
+		rows = await engine.search(`id:${note1.id}`);
+		expect(rows.length).toBe(1);
+		expect(rows.map(r=>r.id)).toContain(note1.id);
+
+		rows = await engine.search(`any:1 id:${note1.id} id:${note2.id}`);
+		expect(rows.length).toBe(2);
+		expect(rows.map(r=>r.id)).toContain(note1.id);
+		expect(rows.map(r=>r.id)).toContain(note2.id);
+
+		rows = await engine.search(`any:0 id:${note1.id} id:${note2.id}`);
+		expect(rows.length).toBe(0);
+
+		rows = await engine.search(`-id:${note2.id}`);
+		expect(rows.length).toBe(2);
+		expect(rows.map(r=>r.id)).toContain(note1.id);
+		expect(rows.map(r=>r.id)).toContain(note3.id);
+	}));
+
 });

packages/lib/JoplinDatabase.ts

@@ -343,7 +343,7 @@ export default class JoplinDatabase extends Database {
 		// must be set in the synchronizer too.
 
 		// Note: v16 and v17 don't do anything. They were used to debug an issue.
-		const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34];
+		const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35];
 
 		let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
 
@@ -864,6 +864,12 @@ export default class JoplinDatabase extends Database {
 				queries.push('CREATE VIRTUAL TABLE notes_spellfix USING spellfix1');
 			}
 
+			if (targetVersion == 35) {
+				queries.push('ALTER TABLE notes_normalized ADD COLUMN todo_due INT NOT NULL DEFAULT 0');
+				queries.push('CREATE INDEX notes_normalized_todo_due ON notes_normalized (todo_due)');
+				queries.push(this.addMigrationFile(35));
+			}
+
 			const updateVersionQuery = { sql: 'UPDATE version SET version = ?', params: [targetVersion] };
 
 			queries.push(updateVersionQuery);

packages/lib/migrations/35.js

@@ -0,0 +1,9 @@
+const SearchEngine = require('../services/searchengine/SearchEngine').default;
+
+const script = {};
+
+script.exec = async function() {
+	await SearchEngine.instance().rebuildIndex();
+};
+
+module.exports = script;

packages/lib/models/Migration.ts

@@ -4,6 +4,7 @@ const migrationScripts: Record<number, any> = {
 	20: require('../migrations/20.js'),
 	27: require('../migrations/27.js'),
 	33: require('../migrations/33.js'),
+	35: require('../migrations/35.js'),
 };
 
 export default class Migration extends BaseModel {

packages/lib/services/searchengine/SearchEngine.ts

@@ -14,7 +14,7 @@ const { pregQuote, scriptType, removeDiacritics } = require('../../string-utils.
 export default class SearchEngine {
 
 	public static instance_: SearchEngine = null;
-	public static relevantFields = 'id, title, body, user_created_time, user_updated_time, is_todo, todo_completed, parent_id, latitude, longitude, altitude, source_url';
+	public static relevantFields = 'id, title, body, user_created_time, user_updated_time, is_todo, todo_completed, todo_due, parent_id, latitude, longitude, altitude, source_url';
 	public static SEARCH_TYPE_AUTO = 'auto';
 	public static SEARCH_TYPE_BASIC = 'basic';
 	public static SEARCH_TYPE_FTS = 'fts';
@@ -82,8 +82,8 @@ export default class SearchEngine {
 				const n = this.normalizeNote_(note);
 				queries.push({ sql: `
 				INSERT INTO notes_normalized(${SearchEngine.relevantFields})
-				VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-				params: [n.id, n.title, n.body, n.user_created_time, n.user_updated_time, n.is_todo, n.todo_completed, n.parent_id, n.latitude, n.longitude, n.altitude, n.source_url] }
+				VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+				params: [n.id, n.title, n.body, n.user_created_time, n.user_updated_time, n.is_todo, n.todo_completed, n.todo_due, n.parent_id, n.latitude, n.longitude, n.altitude, n.source_url] }
 				);
 			}
 
@@ -171,8 +171,8 @@ export default class SearchEngine {
 							const n = this.normalizeNote_(note);
 							queries.push({ sql: `
 							INSERT INTO notes_normalized(${SearchEngine.relevantFields})
-							VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-							params: [change.item_id, n.title, n.body, n.user_created_time, n.user_updated_time, n.is_todo, n.todo_completed, n.parent_id, n.latitude, n.longitude, n.altitude, n.source_url] });
+							VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+							params: [change.item_id, n.title, n.body, n.user_created_time, n.user_updated_time, n.is_todo, n.todo_completed, n.todo_due, n.parent_id, n.latitude, n.longitude, n.altitude, n.source_url] });
 							report.inserted++;
 						}
 					} else if (change.type === ItemChange.TYPE_DELETE) {

packages/lib/services/searchengine/filterParser.ts

@@ -68,8 +68,8 @@ const getTerms = (query: string): Term[] => {
 const parseQuery = (query: string): Term[] => {
 	const validFilters = new Set(['any', 'title', 'body', 'tag',
 		'notebook', 'created', 'updated', 'type',
-		'iscompleted', 'latitude', 'longitude',
-		'altitude', 'resource', 'sourceurl']);
+		'iscompleted', 'due', 'latitude', 'longitude',
+		'altitude', 'resource', 'sourceurl', 'id']);
 
 	const terms = getTerms(query);
 

packages/lib/services/searchengine/queryBuilder.ts

@@ -219,16 +219,24 @@ const genericFilter = (terms: Term[], conditions: string[], params: string[], re
 	}
 
 	const getCondition = (term: Term) => {
-		if (fieldName === 'sourceurl') { return `notes_normalized.source_url ${term.negated ? 'NOT' : ''} LIKE ?`; } else { return `notes_normalized.${fieldName === 'date' ? `user_${term.name}_time` : `${term.name}`} ${term.negated ? '<' : '>='} ?`; }
+		if (fieldName === 'sourceurl') {
+			return `notes_normalized.source_url ${term.negated ? 'NOT' : ''} LIKE ?`;
+		} else if (fieldName === 'date' && term.name === 'due') {
+			return `todo_due ${term.negated ? '<' : '>='} ?`;
+		} else if (fieldName === 'id') {
+			return `id ${term.negated ? 'NOT' : ''} LIKE ?`;
+		} else {
+			return `notes_normalized.${fieldName === 'date' ? `user_${term.name}_time` : `${term.name}`} ${term.negated ? '<' : '>='} ?`;
+		}
 	};
 
 	terms.forEach(term => {
 		conditions.push(`
-		${relation} ROWID IN (
+		${relation} ( ${term.name === 'due' ? 'is_todo IS 1 AND ' : ''} ROWID IN (
 			SELECT ROWID
 			FROM notes_normalized
 			WHERE ${getCondition(term)}
-		)`);
+		))`);
 		params.push(term.value);
 	});
 };
@@ -264,6 +272,12 @@ const biConditionalFilter = (terms: Term[], conditions: string[], relation: Rela
 	});
 };
 
+const noteIdFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation) => {
+	const noteIdTerms = terms.filter(x => x.name === 'id');
+	genericFilter(noteIdTerms, conditions, params, relation, 'id');
+};
+
+
 const typeFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation) => {
 	const typeTerms = terms.filter(x => x.name === 'type');
 	genericFilter(typeTerms, conditions, params, relation, 'type');
@@ -285,7 +299,7 @@ const dateFilter = (terms: Term[], conditons: string[], params: string[], relati
 		const yyyymmdd = /^[0-9]{8}$/;
 		const yyyymm = /^[0-9]{6}$/;
 		const yyyy = /^[0-9]{4}$/;
-		const smartValue = /^(day|week|month|year)-([0-9]+)$/i;
+		const smartValue = /^(day|week|month|year)(\+|-)([0-9]+)$/i;
 
 		if (yyyymmdd.test(date)) {
 			return time.formatLocalToMs(date, 'YYYYMMDD').toString();
@@ -296,14 +310,16 @@ const dateFilter = (terms: Term[], conditons: string[], params: string[], relati
 		} else if (smartValue.test(date)) {
 			const match = smartValue.exec(date);
 			const timeUnit = match[1]; // eg. day, week, month, year
-			const num = Number(match[2]); // eg. 1, 12, 15
-			return time.goBackInTime(Date.now(), num, timeUnit);
+			const timeDirection = match[2]; // + or -
+			const num = Number(match[3]); // eg. 1, 12, 15
+
+			if (timeDirection === '+') { return time.goForwardInTime(Date.now(), num, timeUnit); } else { return time.goBackInTime(Date.now(), num, timeUnit); }
 		} else {
 			throw new Error('Invalid date format!');
 		}
 	};
 
-	const dateTerms = terms.filter(x => x.name === 'created' || x.name === 'updated');
+	const dateTerms = terms.filter(x => x.name === 'created' || x.name === 'updated' || x.name === 'due');
 	const unixDateTerms = dateTerms.map(term => { return { ...term, value: getUnixMs(term.value) }; });
 	genericFilter(unixDateTerms, conditons, params, relation, 'date');
 };
@@ -409,6 +425,7 @@ export default function queryBuilder(terms: Term[]) {
 	FROM notes_fts
 	WHERE ${getConnective(terms, relation)}`);
 
+	noteIdFilter(terms, queryParts, params, relation);
 
 	notebookFilter(terms, queryParts, params, withs);