Api: Resolves #5199: Add support for "events" end point to retrieve info about latest note changes

.eslintignore

@@ -990,6 +990,9 @@ packages/lib/models/Folder.test.js.map
 packages/lib/models/ItemChange.d.ts
 packages/lib/models/ItemChange.js
 packages/lib/models/ItemChange.js.map
+packages/lib/models/ItemChange.test.d.ts
+packages/lib/models/ItemChange.test.js
+packages/lib/models/ItemChange.test.js.map
 packages/lib/models/MasterKey.d.ts
 packages/lib/models/MasterKey.js
 packages/lib/models/MasterKey.js.map
@@ -1404,6 +1407,12 @@ packages/lib/services/rest/actionApi.desktop.js.map
 packages/lib/services/rest/routes/auth.d.ts
 packages/lib/services/rest/routes/auth.js
 packages/lib/services/rest/routes/auth.js.map
+packages/lib/services/rest/routes/events.d.ts
+packages/lib/services/rest/routes/events.js
+packages/lib/services/rest/routes/events.js.map
+packages/lib/services/rest/routes/events.test.d.ts
+packages/lib/services/rest/routes/events.test.js
+packages/lib/services/rest/routes/events.test.js.map
 packages/lib/services/rest/routes/folders.d.ts
 packages/lib/services/rest/routes/folders.js
 packages/lib/services/rest/routes/folders.js.map

.gitignore

@@ -975,6 +975,9 @@ packages/lib/models/Folder.test.js.map
 packages/lib/models/ItemChange.d.ts
 packages/lib/models/ItemChange.js
 packages/lib/models/ItemChange.js.map
+packages/lib/models/ItemChange.test.d.ts
+packages/lib/models/ItemChange.test.js
+packages/lib/models/ItemChange.test.js.map
 packages/lib/models/MasterKey.d.ts
 packages/lib/models/MasterKey.js
 packages/lib/models/MasterKey.js.map
@@ -1389,6 +1392,12 @@ packages/lib/services/rest/actionApi.desktop.js.map
 packages/lib/services/rest/routes/auth.d.ts
 packages/lib/services/rest/routes/auth.js
 packages/lib/services/rest/routes/auth.js.map
+packages/lib/services/rest/routes/events.d.ts
+packages/lib/services/rest/routes/events.js
+packages/lib/services/rest/routes/events.js.map
+packages/lib/services/rest/routes/events.test.d.ts
+packages/lib/services/rest/routes/events.test.js
+packages/lib/services/rest/routes/events.test.js.map
 packages/lib/services/rest/routes/folders.d.ts
 packages/lib/services/rest/routes/folders.js
 packages/lib/services/rest/routes/folders.js.map

packages/app-cli/app/command-apidoc.js

@@ -381,6 +381,30 @@ async function fetchAllNotes() {
 			}
 		}
 
+		{
+			const tableFields = reg.db().tableFields('item_changes', { includeDescription: true });
+
+			lines.push('# Events');
+			lines.push('');
+			lines.push('This end point can be used to retrieve the latest note changes. Currently only note changes are tracked.');
+			lines.push('');
+			lines.push('## Properties');
+			lines.push('');
+			lines.push(this.createPropertiesTable(tableFields));
+			lines.push('');
+			lines.push('## GET /events');
+			lines.push('');
+			lines.push('Returns a paginated list of recent events. A `cursor` property should be provided, which tells from what point in time the events should be returned. The API will return a `cursor` property, to tell from where to resume retrieving events, as well as an `has_more` (tells if more changes can be retrieved) and `items` property, which will contain the list of events. Events are kept for up to 90 days.');
+			lines.push('');
+			lines.push('If no `cursor` property is provided, the API will respond with the latest change ID. That can be used to retrieve future events later on.');
+			lines.push('');
+			lines.push('The results are paginated so will need to may multiple calls to retrieve all the events. Use the `has_more` property to know if more can be retrieved.');
+			lines.push('');
+			lines.push('## GET /events/:id');
+			lines.push('');
+			lines.push('Returns the event with the given ID.');
+		}
+
 		const outFilePath = args['file'];
 
 		await shim.fsDriver().writeFile(outFilePath, lines.join('\n'), 'utf8');

packages/lib/JoplinDatabase.ts

@@ -259,6 +259,14 @@ export default class JoplinDatabase extends Database {
 				folders: {},
 				resources: {},
 				tags: {},
+				item_changes: {
+					type: 'The type of change - either 1 (created), 2 (updated) or 3 (deleted)',
+					created_time: 'When the event was generated',
+					item_type: 'The item type (see table above for the list of item types)',
+					item_id: 'The item ID',
+					before_change_item: 'Unused',
+					source: 'Unused',
+				},
 			};
 
 			const baseItems = ['notes', 'folders', 'tags', 'resources'];

packages/lib/database.ts

@@ -77,7 +77,7 @@ export default class Database {
 		throw new Error(`Invalid field format: ${field}`);
 	}
 
-	escapeFields(fields: string[] | string): string[] | string {
+	public escapeFields(fields: string[] | string): string[] | string {
 		if (fields == '*') return '*';
 
 		const output = [];
@@ -87,6 +87,16 @@ export default class Database {
 		return output;
 	}
 
+	public escapeFieldsToString(fields: string[] | string): string {
+		if (fields === '*') return '*';
+
+		const output = [];
+		for (let i = 0; i < fields.length; i++) {
+			output.push(this.escapeField(fields[i]));
+		}
+		return output.join(',');
+	}
+
 	async tryCall(callName: string, inputSql: StringOrSqlQuery, inputParams: SqlParams) {
 		let sql: string = null;
 		let params: SqlParams = null;

packages/lib/models/ItemChange.test.ts

@@ -1,13 +1,13 @@
-const { revisionService, setupDatabaseAndSynchronizer, db, switchClient } = require('../testing/test-utils.js');
+import { revisionService, setupDatabaseAndSynchronizer, db, switchClient, msleep } from '../testing/test-utils';
-const SearchEngine = require('../services/searchengine/SearchEngine').default;
+import SearchEngine from '../services/searchengine/SearchEngine';
-const ResourceService = require('../services/ResourceService').default;
+import ResourceService from '../services/ResourceService';
-const ItemChangeUtils = require('../services/ItemChangeUtils').default;
+import ItemChangeUtils from '../services/ItemChangeUtils';
-const Note = require('../models/Note').default;
+import Note from '../models/Note';
-const ItemChange = require('../models/ItemChange').default;
+import ItemChange from '../models/ItemChange';
 
-let searchEngine = null;
+let searchEngine: SearchEngine = null;
 
-describe('models_ItemChange', function() {
+describe('models/ItemChange', function() {
 
 	beforeEach(async (done) => {
 		await setupDatabaseAndSynchronizer(1);
@@ -27,17 +27,28 @@ describe('models_ItemChange', function() {
 		const resourceService = new ResourceService();
 
 		await searchEngine.syncTables();
+
 		// If we run this now, it should not delete any change because
 		// the resource service has not yet processed the change
-		await ItemChangeUtils.deleteProcessedChanges();
+		await ItemChangeUtils.deleteProcessedChanges(0);
 		expect(await ItemChange.lastChangeId()).toBe(1);
 
 		await resourceService.indexNoteResources();
-		await ItemChangeUtils.deleteProcessedChanges();
+		await ItemChangeUtils.deleteProcessedChanges(0);
 		expect(await ItemChange.lastChangeId()).toBe(1);
 
 		await revisionService().collectRevisions();
+
+		// If we don't set a TTL it will default to 90 days so it won't delete
+		// either.
 		await ItemChangeUtils.deleteProcessedChanges();
+		expect(await ItemChange.lastChangeId()).toBe(1);
+
+		// All changes should be at least 4 ms old now
+		await msleep(4);
+
+		// Now it should delete all changes older than 3 ms
+		await ItemChangeUtils.deleteProcessedChanges(3);
 		expect(await ItemChange.lastChangeId()).toBe(0);
 	}));
 

packages/lib/models/ItemChange.ts

@@ -1,8 +1,14 @@
 import BaseModel, { ModelType } from '../BaseModel';
 import shim from '../shim';
 import eventManager from '../eventManager';
+import { ItemChangeEntity } from '../services/database/types';
 const Mutex = require('async-mutex').Mutex;
 
+export interface ChangeSinceIdOptions {
+	limit?: number;
+	fields?: string[];
+}
+
 export default class ItemChange extends BaseModel {
 
 	private static addChangeMutex_: any = new Mutex();
@@ -24,7 +30,7 @@ export default class ItemChange extends BaseModel {
 		return BaseModel.TYPE_ITEM_CHANGE;
 	}
 
-	static async add(itemType: ModelType, itemId: string, type: number, changeSource: any = null, beforeChangeItemJson: string = null) {
+	public static async add(itemType: ModelType, itemId: string, type: number, changeSource: any = null, beforeChangeItemJson: string = null) {
 		if (changeSource === null) changeSource = ItemChange.SOURCE_UNSPECIFIED;
 		if (!beforeChangeItemJson) beforeChangeItemJson = '';
 
@@ -57,14 +63,14 @@ export default class ItemChange extends BaseModel {
 		}
 	}
 
-	static async lastChangeId() {
+	public static async lastChangeId() {
 		const row = await this.db().selectOne('SELECT max(id) as max_id FROM item_changes');
 		return row && row.max_id ? row.max_id : 0;
 	}
 
 	// Because item changes are recorded in the background, this function
 	// can be used for synchronous code, in particular when unit testing.
-	static async waitForAllSaved() {
+	public static async waitForAllSaved() {
 		return new Promise((resolve) => {
 			const iid = shim.setInterval(() => {
 				if (!ItemChange.saveCalls_.length) {
@@ -75,8 +81,32 @@ export default class ItemChange extends BaseModel {
 		});
 	}
 
-	static async deleteOldChanges(lowestChangeId: number) {
+	public static async deleteOldChanges(lowestChangeId: number, itemMinTtl: number) {
 		if (!lowestChangeId) return;
-		return this.db().exec('DELETE FROM item_changes WHERE id <= ?', [lowestChangeId]);
+
+		const cutOffDate = Date.now() - itemMinTtl;
+
+		return this.db().exec(`
+			DELETE FROM item_changes
+			WHERE id <= ?
+			AND created_time <= ?
+		`, [lowestChangeId, cutOffDate]);
 	}
+
+	public static async changesSinceId(changeId: number, options: ChangeSinceIdOptions = null): Promise<ItemChangeEntity[]> {
+		options = {
+			limit: 100,
+			fields: ['id', 'item_type', 'item_id', 'type', 'created_time'],
+			...options,
+		};
+
+		return this.db().selectAll(`
+			SELECT ${this.db().escapeFieldsToString(options.fields)}
+			FROM item_changes
+			WHERE id > ?
+			ORDER BY id
+			LIMIT ?
+		`, [changeId, options.limit]);
+	}
+
 }

packages/lib/services/ItemChangeUtils.ts

@@ -1,8 +1,10 @@
 import Setting from '../models/Setting';
 import ItemChange from '../models/ItemChange';
 
+const dayMs = 86400000;
+
 export default class ItemChangeUtils {
-	static async deleteProcessedChanges() {
+	static async deleteProcessedChanges(itemMinTtl: number = dayMs * 90) {
 		const lastProcessedChangeIds = [
 			Setting.value('resourceService.lastProcessedChangeId'),
 			Setting.value('searchEngine.lastProcessedChangeId'),
@@ -10,6 +12,6 @@ export default class ItemChangeUtils {
 		];
 
 		const lowestChangeId = Math.min(...lastProcessedChangeIds);
-		await ItemChange.deleteOldChanges(lowestChangeId);
+		await ItemChange.deleteOldChanges(lowestChangeId, itemMinTtl);
 	}
 }

packages/lib/services/rest/Api.ts

@@ -9,6 +9,7 @@ import route_master_keys from './routes/master_keys';
 import route_search from './routes/search';
 import route_ping from './routes/ping';
 import route_auth from './routes/auth';
+import route_events from './routes/events';
 
 const { ltrimSlashes } = require('../../path-utils');
 const md5 = require('md5');
@@ -43,6 +44,9 @@ interface RequestQuery {
 
 	// Auth token
 	auth_token?: string;
+
+	// Event cursor
+	cursor?: string;
 }
 
 export interface Request {
@@ -104,6 +108,7 @@ export default class Api {
 			search: route_search,
 			services: this.action_services.bind(this),
 			auth: route_auth,
+			events: route_events,
 		};
 
 		this.dispatch = this.dispatch.bind(this);

packages/lib/services/rest/routes/events.test.ts

@@ -0,0 +1,94 @@
+import { ModelType } from '../../../BaseModel';
+import ItemChange from '../../../models/ItemChange';
+import Note from '../../../models/Note';
+import { expectThrow, setupDatabaseAndSynchronizer, switchClient } from '../../../testing/test-utils';
+import { ItemChangeEntity } from '../../database/types';
+import Api, { RequestMethod } from '../Api';
+
+let api: Api = null;
+
+describe('routes/events', function() {
+
+	beforeEach(async (done) => {
+		api = new Api();
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+		done();
+	});
+
+	it('should retrieve the latest events', async () => {
+		let cursor = '0';
+
+		{
+			const response = await api.route(RequestMethod.GET, 'events', { cursor });
+			expect(response.cursor).toBe('0');
+		}
+
+		const note1 = await Note.save({ title: 'toto' });
+		await Note.save({ id: note1.id, title: 'tutu' });
+		const note2 = await Note.save({ title: 'tata' });
+		await ItemChange.waitForAllSaved();
+
+		{
+			const response = await api.route(RequestMethod.GET, 'events', { cursor });
+			expect(response.cursor).toBe('3');
+			expect(response.items.length).toBe(2);
+			expect(response.has_more).toBe(false);
+			expect(response.items.map((it: ItemChangeEntity) => it.item_id).sort()).toEqual([note1.id, note2.id].sort());
+
+			cursor = response.cursor;
+		}
+
+		{
+			const response = await api.route(RequestMethod.GET, 'events', { cursor });
+			expect(response.cursor).toBe(cursor);
+			expect(response.items.length).toBe(0);
+			expect(response.has_more).toBe(false);
+		}
+
+		await Note.save({ id: note2.id, title: 'titi' });
+		await ItemChange.waitForAllSaved();
+
+		{
+			const response = await api.route(RequestMethod.GET, 'events', { cursor });
+			expect(response.cursor).toBe('4');
+			expect(response.items.length).toBe(1);
+			expect(response.items[0].item_id).toBe(note2.id);
+		}
+	});
+
+	it('should limit the number of response items', async () => {
+		const promises = [];
+		for (let i = 0; i < 101; i++) {
+			promises.push(Note.save({ title: 'toto' }));
+		}
+
+		await Promise.all(promises);
+		await ItemChange.waitForAllSaved();
+
+		const response1 = await api.route(RequestMethod.GET, 'events', { cursor: '0' });
+		expect(response1.items.length).toBe(100);
+		expect(response1.has_more).toBe(true);
+
+		const response2 = await api.route(RequestMethod.GET, 'events', { cursor: response1.cursor });
+		expect(response2.items.length).toBe(1);
+		expect(response2.has_more).toBe(false);
+	});
+
+	it('should retrieve a single item', async () => {
+		const beforeTime = Date.now();
+
+		const note = await Note.save({ title: 'toto' });
+		await ItemChange.waitForAllSaved();
+
+		const response = await api.route(RequestMethod.GET, 'events/1');
+
+		expect(response.item_type).toBe(ModelType.Note);
+		expect(response.type).toBe(1);
+		expect(response.item_id).toBe(note.id);
+		expect(response.created_time).toBeGreaterThanOrEqual(beforeTime);
+
+		await expectThrow(async () => api.route(RequestMethod.GET, 'events/1234'));
+	});
+
+});

packages/lib/services/rest/routes/events.ts

@@ -0,0 +1,39 @@
+import { ModelType } from '../../../BaseModel';
+import { Request, RequestMethod } from '../Api';
+import { ErrorBadRequest, ErrorNotFound } from '../utils/errors';
+import ItemChange, { ChangeSinceIdOptions } from '../../../models/ItemChange';
+import requestFields from '../utils/requestFields';
+
+export default async function(request: Request, id: string = null, _link: string = null) {
+	if (request.method === RequestMethod.GET) {
+		const options: ChangeSinceIdOptions = {
+			limit: 100,
+			fields: requestFields(request, ModelType.ItemChange, ['id', 'item_type', 'item_id', 'type', 'created_time']),
+		};
+
+		if (!id) {
+			if (!('cursor' in request.query)) {
+				return {
+					items: [],
+					has_more: false,
+					cursor: (await ItemChange.lastChangeId()).toString(),
+				};
+			} else {
+				const cursor = Number(request.query.cursor);
+				if (isNaN(cursor)) throw new ErrorBadRequest(`Invalid cursor: ${request.query.cursor}`);
+
+				const changes = await ItemChange.changesSinceId(cursor, options);
+
+				return {
+					items: changes,
+					has_more: changes.length >= options.limit,
+					cursor: (changes.length ? changes[changes.length - 1].id : cursor).toString(),
+				};
+			}
+		} else {
+			const change = await ItemChange.load(id, { fields: options.fields });
+			if (!change) throw new ErrorNotFound();
+			return change;
+		}
+	}
+}

packages/lib/services/rest/utils/requestFields.ts

@@ -11,13 +11,18 @@ function defaultFieldsByModelType(modelType: number): string[] {
 	return output;
 }
 
-export default function(request: Request, modelType: number) {
+export default function(request: Request, modelType: number, defaultFields: string[] = null) {
+	const getDefaults = () => {
+		if (defaultFields) return defaultFields;
+		return defaultFieldsByModelType(modelType);
+	};
+
 	const query = request.query;
-	if (!query || !query.fields) return defaultFieldsByModelType(modelType);
+	if (!query || !query.fields) return getDefaults();
 	if (Array.isArray(query.fields)) return query.fields.slice();
 	const fields = query.fields
 		.split(',')
 		.map((f: string) => f.trim())
 		.filter((f: string) => !!f);
-	return fields.length ? fields : defaultFieldsByModelType(modelType);
+	return fields.length ? fields : getDefaults();
 }