self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-01-08 13:06:15 +02:00
Code Issues Releases Activity

Api: Resolves #5199: Add support for "events" end point to retrieve info about latest note changes

Browse Source
This commit is contained in:
Laurent Cozic 2021-08-30 18:53:24 +01:00
parent ce89ee5bab
commit b88b747ba6
12 changed files with 267 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.eslintignore
View File

@ -990,6 +990,9 @@ packages/lib/models/Folder.test.js.map
packages/lib/models/ItemChange.d.ts
packages/lib/models/ItemChange.js
packages/lib/models/ItemChange.js.map
packages/lib/models/ItemChange.test.d.ts
packages/lib/models/ItemChange.test.js
packages/lib/models/ItemChange.test.js.map
packages/lib/models/MasterKey.d.ts
packages/lib/models/MasterKey.js
packages/lib/models/MasterKey.js.map
@ -1404,6 +1407,12 @@ packages/lib/services/rest/actionApi.desktop.js.map
packages/lib/services/rest/routes/auth.d.ts
packages/lib/services/rest/routes/auth.js
packages/lib/services/rest/routes/auth.js.map
packages/lib/services/rest/routes/events.d.ts
packages/lib/services/rest/routes/events.js
packages/lib/services/rest/routes/events.js.map
packages/lib/services/rest/routes/events.test.d.ts
packages/lib/services/rest/routes/events.test.js
packages/lib/services/rest/routes/events.test.js.map
packages/lib/services/rest/routes/folders.d.ts
packages/lib/services/rest/routes/folders.js
packages/lib/services/rest/routes/folders.js.map

9
.gitignore vendored
View File

@ -975,6 +975,9 @@ packages/lib/models/Folder.test.js.map
packages/lib/models/ItemChange.d.ts
packages/lib/models/ItemChange.js
packages/lib/models/ItemChange.js.map
packages/lib/models/ItemChange.test.d.ts
packages/lib/models/ItemChange.test.js
packages/lib/models/ItemChange.test.js.map
packages/lib/models/MasterKey.d.ts
packages/lib/models/MasterKey.js
packages/lib/models/MasterKey.js.map
@ -1389,6 +1392,12 @@ packages/lib/services/rest/actionApi.desktop.js.map
packages/lib/services/rest/routes/auth.d.ts
packages/lib/services/rest/routes/auth.js
packages/lib/services/rest/routes/auth.js.map
packages/lib/services/rest/routes/events.d.ts
packages/lib/services/rest/routes/events.js
packages/lib/services/rest/routes/events.js.map
packages/lib/services/rest/routes/events.test.d.ts
packages/lib/services/rest/routes/events.test.js
packages/lib/services/rest/routes/events.test.js.map
packages/lib/services/rest/routes/folders.d.ts
packages/lib/services/rest/routes/folders.js
packages/lib/services/rest/routes/folders.js.map

24
packages/app-cli/app/command-apidoc.js
View File

@ -381,6 +381,30 @@ async function fetchAllNotes() {
}
}
{
const tableFields = reg.db().tableFields('item_changes', { includeDescription: true });
lines.push('# Events');
lines.push('');
lines.push('This end point can be used to retrieve the latest note changes. Currently only note changes are tracked.');
lines.push('');
lines.push('## Properties');
lines.push('');
lines.push(this.createPropertiesTable(tableFields));
lines.push('');
lines.push('## GET /events');
lines.push('');
lines.push('Returns a paginated list of recent events. A `cursor` property should be provided, which tells from what point in time the events should be returned. The API will return a `cursor` property, to tell from where to resume retrieving events, as well as an `has_more` (tells if more changes can be retrieved) and `items` property, which will contain the list of events. Events are kept for up to 90 days.');
lines.push('');
lines.push('If no `cursor` property is provided, the API will respond with the latest change ID. That can be used to retrieve future events later on.');
lines.push('');
lines.push('The results are paginated so will need to may multiple calls to retrieve all the events. Use the `has_more` property to know if more can be retrieved.');
lines.push('');
lines.push('## GET /events/:id');
lines.push('');
lines.push('Returns the event with the given ID.');
}
const outFilePath = args['file'];
await shim.fsDriver().writeFile(outFilePath, lines.join('\n'), 'utf8');

8
packages/lib/JoplinDatabase.ts
View File

@ -259,6 +259,14 @@ export default class JoplinDatabase extends Database {
folders: {},
resources: {},
tags: {},
item_changes: {
type: 'The type of change - either 1 (created), 2 (updated) or 3 (deleted)',
created_time: 'When the event was generated',
item_type: 'The item type (see table above for the list of item types)',
item_id: 'The item ID',
before_change_item: 'Unused',
source: 'Unused',
},
};
const baseItems = ['notes', 'folders', 'tags', 'resources'];

12
packages/lib/database.ts
View File

@ -77,7 +77,7 @@ export default class Database {
throw new Error(`Invalid field format: ${field}`);
}
escapeFields(fields: string[] | string): string[] | string {
public escapeFields(fields: string[] | string): string[] | string {
if (fields == '*') return '*';
const output = [];
@ -87,6 +87,16 @@ export default class Database {
return output;
}
public escapeFieldsToString(fields: string[] | string): string {
if (fields === '*') return '*';
const output = [];
for (let i = 0; i < fields.length; i++) {
output.push(this.escapeField(fields[i]));
}
return output.join(',');
}
async tryCall(callName: string, inputSql: StringOrSqlQuery, inputParams: SqlParams) {
let sql: string = null;
let params: SqlParams = null;

31
packages/lib/models/ItemChange.test.js → packages/lib/models/ItemChange.test.ts
View File

@ -1,13 +1,13 @@
const { revisionService, setupDatabaseAndSynchronizer, db, switchClient } = require('../testing/test-utils.js');
const SearchEngine = require('../services/searchengine/SearchEngine').default;
const ResourceService = require('../services/ResourceService').default;
const ItemChangeUtils = require('../services/ItemChangeUtils').default;
const Note = require('../models/Note').default;
const ItemChange = require('../models/ItemChange').default;
import { revisionService, setupDatabaseAndSynchronizer, db, switchClient, msleep } from '../testing/test-utils';
import SearchEngine from '../services/searchengine/SearchEngine';
import ResourceService from '../services/ResourceService';
import ItemChangeUtils from '../services/ItemChangeUtils';
import Note from '../models/Note';
import ItemChange from '../models/ItemChange';
let searchEngine = null;
let searchEngine: SearchEngine = null;
describe('models_ItemChange', function() {
describe('models/ItemChange', function() {
beforeEach(async (done) => {
await setupDatabaseAndSynchronizer(1);
@ -27,17 +27,28 @@ describe('models_ItemChange', function() {
const resourceService = new ResourceService();
await searchEngine.syncTables();
// If we run this now, it should not delete any change because
// the resource service has not yet processed the change
await ItemChangeUtils.deleteProcessedChanges();
await ItemChangeUtils.deleteProcessedChanges(0);
expect(await ItemChange.lastChangeId()).toBe(1);
await resourceService.indexNoteResources();
await ItemChangeUtils.deleteProcessedChanges();
await ItemChangeUtils.deleteProcessedChanges(0);
expect(await ItemChange.lastChangeId()).toBe(1);
await revisionService().collectRevisions();
// If we don't set a TTL it will default to 90 days so it won't delete
// either.
await ItemChangeUtils.deleteProcessedChanges();
expect(await ItemChange.lastChangeId()).toBe(1);
// All changes should be at least 4 ms old now
await msleep(4);
// Now it should delete all changes older than 3 ms
await ItemChangeUtils.deleteProcessedChanges(3);
expect(await ItemChange.lastChangeId()).toBe(0);
}));

40
packages/lib/models/ItemChange.ts
View File

@ -1,8 +1,14 @@
import BaseModel, { ModelType } from '../BaseModel';
import shim from '../shim';
import eventManager from '../eventManager';
import { ItemChangeEntity } from '../services/database/types';
const Mutex = require('async-mutex').Mutex;
export interface ChangeSinceIdOptions {
limit?: number;
fields?: string[];
}
export default class ItemChange extends BaseModel {
private static addChangeMutex_: any = new Mutex();
@ -24,7 +30,7 @@ export default class ItemChange extends BaseModel {
return BaseModel.TYPE_ITEM_CHANGE;
}
static async add(itemType: ModelType, itemId: string, type: number, changeSource: any = null, beforeChangeItemJson: string = null) {
public static async add(itemType: ModelType, itemId: string, type: number, changeSource: any = null, beforeChangeItemJson: string = null) {
if (changeSource === null) changeSource = ItemChange.SOURCE_UNSPECIFIED;
if (!beforeChangeItemJson) beforeChangeItemJson = '';
@ -57,14 +63,14 @@ export default class ItemChange extends BaseModel {
}
}
static async lastChangeId() {
public static async lastChangeId() {
const row = await this.db().selectOne('SELECT max(id) as max_id FROM item_changes');
return row && row.max_id ? row.max_id : 0;
}
// Because item changes are recorded in the background, this function
// can be used for synchronous code, in particular when unit testing.
static async waitForAllSaved() {
public static async waitForAllSaved() {
return new Promise((resolve) => {
const iid = shim.setInterval(() => {
if (!ItemChange.saveCalls_.length) {
@ -75,8 +81,32 @@ export default class ItemChange extends BaseModel {
});
}
static async deleteOldChanges(lowestChangeId: number) {
public static async deleteOldChanges(lowestChangeId: number, itemMinTtl: number) {
if (!lowestChangeId) return;
return this.db().exec('DELETE FROM item_changes WHERE id <= ?', [lowestChangeId]);
const cutOffDate = Date.now() - itemMinTtl;
return this.db().exec(`
DELETE FROM item_changes
WHERE id <= ?
AND created_time <= ?
`, [lowestChangeId, cutOffDate]);
}
public static async changesSinceId(changeId: number, options: ChangeSinceIdOptions = null): Promise<ItemChangeEntity[]> {
options = {
limit: 100,
fields: ['id', 'item_type', 'item_id', 'type', 'created_time'],
...options,
};
return this.db().selectAll(`
SELECT ${this.db().escapeFieldsToString(options.fields)}
FROM item_changes
WHERE id > ?
ORDER BY id
LIMIT ?
`, [changeId, options.limit]);
}
}

6
packages/lib/services/ItemChangeUtils.ts
View File

@ -1,8 +1,10 @@
import Setting from '../models/Setting';
import ItemChange from '../models/ItemChange';
const dayMs = 86400000;
export default class ItemChangeUtils {
static async deleteProcessedChanges() {
static async deleteProcessedChanges(itemMinTtl: number = dayMs * 90) {
const lastProcessedChangeIds = [
Setting.value('resourceService.lastProcessedChangeId'),
Setting.value('searchEngine.lastProcessedChangeId'),
@ -10,6 +12,6 @@ export default class ItemChangeUtils {
];
const lowestChangeId = Math.min(...lastProcessedChangeIds);
await ItemChange.deleteOldChanges(lowestChangeId);
await ItemChange.deleteOldChanges(lowestChangeId, itemMinTtl);
}
}

5
packages/lib/services/rest/Api.ts
View File

@ -9,6 +9,7 @@ import route_master_keys from './routes/master_keys';
import route_search from './routes/search';
import route_ping from './routes/ping';
import route_auth from './routes/auth';
import route_events from './routes/events';
const { ltrimSlashes } = require('../../path-utils');
const md5 = require('md5');
@ -43,6 +44,9 @@ interface RequestQuery {
// Auth token
auth_token?: string;
// Event cursor
cursor?: string;
}
export interface Request {
@ -104,6 +108,7 @@ export default class Api {
search: route_search,
services: this.action_services.bind(this),
auth: route_auth,
events: route_events,
};
this.dispatch = this.dispatch.bind(this);

94
packages/lib/services/rest/routes/events.test.ts Normal file
View File

@ -0,0 +1,94 @@
import { ModelType } from '../../../BaseModel';
import ItemChange from '../../../models/ItemChange';
import Note from '../../../models/Note';
import { expectThrow, setupDatabaseAndSynchronizer, switchClient } from '../../../testing/test-utils';
import { ItemChangeEntity } from '../../database/types';
import Api, { RequestMethod } from '../Api';
let api: Api = null;
describe('routes/events', function() {
beforeEach(async (done) => {
api = new Api();
await setupDatabaseAndSynchronizer(1);
await switchClient(1);
done();
});
it('should retrieve the latest events', async () => {
let cursor = '0';
{
const response = await api.route(RequestMethod.GET, 'events', { cursor });
expect(response.cursor).toBe('0');
}
const note1 = await Note.save({ title: 'toto' });
await Note.save({ id: note1.id, title: 'tutu' });
const note2 = await Note.save({ title: 'tata' });
await ItemChange.waitForAllSaved();
{
const response = await api.route(RequestMethod.GET, 'events', { cursor });
expect(response.cursor).toBe('3');
expect(response.items.length).toBe(2);
expect(response.has_more).toBe(false);
expect(response.items.map((it: ItemChangeEntity) => it.item_id).sort()).toEqual([note1.id, note2.id].sort());
cursor = response.cursor;
}
{
const response = await api.route(RequestMethod.GET, 'events', { cursor });
expect(response.cursor).toBe(cursor);
expect(response.items.length).toBe(0);
expect(response.has_more).toBe(false);
}
await Note.save({ id: note2.id, title: 'titi' });
await ItemChange.waitForAllSaved();
{
const response = await api.route(RequestMethod.GET, 'events', { cursor });
expect(response.cursor).toBe('4');
expect(response.items.length).toBe(1);
expect(response.items[0].item_id).toBe(note2.id);
}
});
it('should limit the number of response items', async () => {
const promises = [];
for (let i = 0; i < 101; i++) {
promises.push(Note.save({ title: 'toto' }));
}
await Promise.all(promises);
await ItemChange.waitForAllSaved();
const response1 = await api.route(RequestMethod.GET, 'events', { cursor: '0' });
expect(response1.items.length).toBe(100);
expect(response1.has_more).toBe(true);
const response2 = await api.route(RequestMethod.GET, 'events', { cursor: response1.cursor });
expect(response2.items.length).toBe(1);
expect(response2.has_more).toBe(false);
});
it('should retrieve a single item', async () => {
const beforeTime = Date.now();
const note = await Note.save({ title: 'toto' });
await ItemChange.waitForAllSaved();
const response = await api.route(RequestMethod.GET, 'events/1');
expect(response.item_type).toBe(ModelType.Note);
expect(response.type).toBe(1);
expect(response.item_id).toBe(note.id);
expect(response.created_time).toBeGreaterThanOrEqual(beforeTime);
await expectThrow(async () => api.route(RequestMethod.GET, 'events/1234'));
});
});

39
packages/lib/services/rest/routes/events.ts Normal file
View File

@ -0,0 +1,39 @@
import { ModelType } from '../../../BaseModel';
import { Request, RequestMethod } from '../Api';
import { ErrorBadRequest, ErrorNotFound } from '../utils/errors';
import ItemChange, { ChangeSinceIdOptions } from '../../../models/ItemChange';
import requestFields from '../utils/requestFields';
export default async function(request: Request, id: string = null, _link: string = null) {
if (request.method === RequestMethod.GET) {
const options: ChangeSinceIdOptions = {
limit: 100,
fields: requestFields(request, ModelType.ItemChange, ['id', 'item_type', 'item_id', 'type', 'created_time']),
};
if (!id) {
if (!('cursor' in request.query)) {
return {
items: [],
has_more: false,
cursor: (await ItemChange.lastChangeId()).toString(),
};
} else {
const cursor = Number(request.query.cursor);
if (isNaN(cursor)) throw new ErrorBadRequest(`Invalid cursor: ${request.query.cursor}`);
const changes = await ItemChange.changesSinceId(cursor, options);
return {
items: changes,
has_more: changes.length >= options.limit,
cursor: (changes.length ? changes[changes.length - 1].id : cursor).toString(),
};
}
} else {
const change = await ItemChange.load(id, { fields: options.fields });
if (!change) throw new ErrorNotFound();
return change;
}
}
}

11
packages/lib/services/rest/utils/requestFields.ts
View File

@ -11,13 +11,18 @@ function defaultFieldsByModelType(modelType: number): string[] {
return output;
}
export default function(request: Request, modelType: number) {
export default function(request: Request, modelType: number, defaultFields: string[] = null) {
const getDefaults = () => {
if (defaultFields) return defaultFields;
return defaultFieldsByModelType(modelType);
};
const query = request.query;
if (!query || !query.fields) return defaultFieldsByModelType(modelType);
if (!query || !query.fields) return getDefaults();
if (Array.isArray(query.fields)) return query.fields.slice();
const fields = query.fields
.split(',')
.map((f: string) => f.trim())
.filter((f: string) => !!f);
return fields.length ? fields : defaultFieldsByModelType(modelType);
return fields.length ? fields : getDefaults();
}