All: Fixes #5007: Items are filtered in the API search (#5017)

packages/app-mobile/components/screens/search.js

@@ -105,7 +105,7 @@ class SearchScreenComponent extends BaseScreenComponent {
 
 		if (query) {
 			if (this.props.settings['db.ftsEnabled']) {
-				notes = await SearchEngineUtils.notesForQuery(query);
+				notes = await SearchEngineUtils.notesForQuery(query, true);
 			} else {
 				const p = query.split(' ');
 				const temp = [];

packages/lib/BaseApplication.ts

@@ -313,7 +313,7 @@ export default class BaseApplication {
 				notes = await Tag.notes(parentId, options);
 			} else if (parentType === BaseModel.TYPE_SEARCH) {
 				const search = BaseModel.byId(state.searches, parentId);
-				notes = await SearchEngineUtils.notesForQuery(search.query_pattern);
+				notes = await SearchEngineUtils.notesForQuery(search.query_pattern, true);
 				const parsedQuery = await SearchEngine.instance().parseQuery(search.query_pattern);
 				highlightedWords = SearchEngine.instance().allParsedQueryTerms(parsedQuery);
 			} else if (parentType === BaseModel.TYPE_SMART_FILTER) {

packages/lib/services/rest/routes/search.ts

@@ -28,7 +28,7 @@ export default async function(request: Request) {
 		options.caseInsensitive = true;
 		results = await ModelClass.all(options);
 	} else {
-		results = await SearchEngineUtils.notesForQuery(query, defaultLoadOptions(request, ModelType.Note));
+		results = await SearchEngineUtils.notesForQuery(query, false, defaultLoadOptions(request, ModelType.Note));
 	}
 
 	return collectionToPaginatedResults(modelType, results, request);

packages/lib/services/searchengine/SearchEngineUtils.test.ts

@@ -26,12 +26,21 @@ describe('services_SearchEngineUtils', function() {
 
 			Setting.setValue('showCompletedTodos', true);
 
-			const rows = await SearchEngineUtils.notesForQuery('abcd', null, searchEngine);
+			const rows = await SearchEngineUtils.notesForQuery('abcd', true, null, searchEngine);
 
 			expect(rows.length).toBe(3);
 			expect(rows.map(r=>r.id)).toContain(note1.id);
 			expect(rows.map(r=>r.id)).toContain(todo1.id);
 			expect(rows.map(r=>r.id)).toContain(todo2.id);
+
+			const options: any = {};
+			options.fields = ['id', 'title'];
+
+			const rows2 = await SearchEngineUtils.notesForQuery('abcd', true, options, searchEngine);
+			expect(rows2.length).toBe(3);
+			expect(rows2.map(r=>r.id)).toContain(note1.id);
+			expect(rows2.map(r=>r.id)).toContain(todo1.id);
+			expect(rows2.map(r=>r.id)).toContain(todo2.id);
 		}));
 
 		it('hide completed', (async () => {
@@ -43,11 +52,35 @@ describe('services_SearchEngineUtils', function() {
 
 			Setting.setValue('showCompletedTodos', false);
 
-			const rows = await SearchEngineUtils.notesForQuery('abcd', null, searchEngine);
+			const rows = await SearchEngineUtils.notesForQuery('abcd', true, null, searchEngine);
 
 			expect(rows.length).toBe(2);
 			expect(rows.map(r=>r.id)).toContain(note1.id);
 			expect(rows.map(r=>r.id)).toContain(todo1.id);
+
+			const options: any = {};
+			options.fields = ['id', 'title'];
+			const rows2 = await SearchEngineUtils.notesForQuery('abcd', true, options, searchEngine);
+			expect(rows2.length).toBe(2);
+			expect(rows2.map(r=>r.id)).toContain(note1.id);
+			expect(rows2.map(r=>r.id)).toContain(todo1.id);
+		}));
+
+		it('show completed (!applyUserSettings)', (async () => {
+			const note1 = await Note.save({ title: 'abcd', body: 'body 1' });
+			const todo1 = await Note.save({ title: 'abcd', body: 'todo 1', is_todo: 1 });
+			await Note.save({ title: 'qwer', body: 'body 2' });
+			const todo2 = await Note.save({ title: 'abcd', body: 'todo 2', is_todo: 1, todo_completed: 1590085027710 });
+			await searchEngine.syncTables();
+
+			Setting.setValue('showCompletedTodos', false);
+
+			const rows = await SearchEngineUtils.notesForQuery('abcd', false, null, searchEngine);
+
+			expect(rows.length).toBe(3);
+			expect(rows.map(r=>r.id)).toContain(note1.id);
+			expect(rows.map(r=>r.id)).toContain(todo1.id);
+			expect(rows.map(r=>r.id)).toContain(todo2.id);
 		}));
 	});
 });

packages/lib/services/searchengine/SearchEngineUtils.ts

@@ -3,7 +3,7 @@ import Note from '../../models/Note';
 import Setting from '../../models/Setting';
 
 export default class SearchEngineUtils {
-	static async notesForQuery(query: string, options: any = null, searchEngine: SearchEngine = null) {
+	static async notesForQuery(query: string, applyUserSettings: boolean, options: any = null, searchEngine: SearchEngine = null) {
 		if (!options) options = {};
 
 		if (!searchEngine) {
@@ -30,6 +30,20 @@ export default class SearchEngineUtils {
 			idWasAutoAdded = true;
 		}
 
+		// Add fields is_todo and todo_completed for showCompletedTodos filtering.
+		// Also remember that the field  was auto-added so that it can be removed afterwards.
+		let isTodoAutoAdded = false;
+		if (fields.indexOf('is_todo') < 0) {
+			fields.push('is_todo');
+			isTodoAutoAdded = true;
+		}
+
+		let isTodoCompletedAutoAdded = false;
+		if (fields.indexOf('todo_completed') < 0) {
+			fields.push('todo_completed');
+			isTodoCompletedAutoAdded = true;
+		}
+
 		const previewOptions = Object.assign({}, {
 			order: [],
 			fields: fields,
@@ -38,20 +52,22 @@ export default class SearchEngineUtils {
 
 		const notes = await Note.previews(null, previewOptions);
 
+		// Filter completed todos
+		let filteredNotes = [...notes];
+		if (applyUserSettings && !Setting.value('showCompletedTodos')) {
+			filteredNotes = notes.filter(note => note.is_todo === 0 || (note.is_todo === 1 && note.todo_completed === 0));
+		}
+
 		// By default, the notes will be returned in reverse order
 		// or maybe random order so sort them here in the correct order
 		// (search engine returns the results in order of relevance).
 		const sortedNotes = [];
-		for (let i = 0; i < notes.length; i++) {
+		for (let i = 0; i < filteredNotes.length; i++) {
-			const idx = noteIds.indexOf(notes[i].id);
+			const idx = noteIds.indexOf(filteredNotes[i].id);
-			sortedNotes[idx] = notes[i];
+			sortedNotes[idx] = filteredNotes[i];
 			if (idWasAutoAdded) delete sortedNotes[idx].id;
-		}
+			if (isTodoCompletedAutoAdded) delete sortedNotes[idx].is_todo;
-
+			if (isTodoAutoAdded) delete sortedNotes[idx].todo_completed;
-		// Filter completed todos
-		let filteredNotes = [...sortedNotes];
-		if (!Setting.value('showCompletedTodos')) {
-			filteredNotes = sortedNotes.filter(note => note.is_todo === 0 || (note.is_todo === 1 && note.todo_completed === 0));
 		}
 
 		// Note that when the search engine index is somehow corrupted, it might contain
@@ -60,9 +76,9 @@ export default class SearchEngineUtils {
 		// issue: https://discourse.joplinapp.org/t/how-to-recover-corrupted-database/9367
 		if (noteIds.length !== notes.length) {
 			// remove null objects
-			return filteredNotes.filter(n => n);
+			return sortedNotes.filter(n => n);
 		} else {
-			return filteredNotes;
+			return sortedNotes;
 		}
 
 	}