From f5b5cf74c5f6211d158d67f7c89813ec10fd39b7 Mon Sep 17 00:00:00 2001 From: Adrien Poupa Date: Mon, 9 Dec 2024 10:48:21 -0500 Subject: [PATCH] Server: Resolves #9931: Add task to delete events older than a week (#11372) --- packages/server/src/env.ts | 12 ++++- packages/server/src/models/EventModel.test.ts | 50 ++++++++++++++++++- packages/server/src/models/EventModel.ts | 7 +++ packages/server/src/services/TaskService.ts | 1 + .../server/src/services/database/types.ts | 1 + packages/server/src/utils/setupTaskService.ts | 10 ++++ 6 files changed, 79 insertions(+), 2 deletions(-) diff --git a/packages/server/src/env.ts b/packages/server/src/env.ts index 31b24eaff..e59534851 100644 --- a/packages/server/src/env.ts +++ b/packages/server/src/env.ts @@ -30,7 +30,7 @@ const defaultEnvValues: EnvVariables = { IS_ADMIN_INSTANCE: true, INSTANCE_NAME: '', - // Maxiumm allowed drift between NTP time and server time. A few + // Maximum allowed drift between NTP time and server time. A few // milliseconds is normally not an issue unless many clients are modifying // the same note at the exact same time. But past a certain limit, it might // mean the server clock is incorrect and should be fixed, as that could @@ -118,6 +118,13 @@ const defaultEnvValues: EnvVariables = { USER_DATA_AUTO_DELETE_ENABLED: false, USER_DATA_AUTO_DELETE_AFTER_DAYS: 90, + // ================================================== + // Events deletion + // ================================================== + + EVENTS_AUTO_DELETE_ENABLED: false, + EVENTS_AUTO_DELETE_AFTER_DAYS: 30, + // ================================================== // LDAP configuration // ================================================== @@ -210,6 +217,9 @@ export interface EnvVariables { USER_DATA_AUTO_DELETE_ENABLED: boolean; USER_DATA_AUTO_DELETE_AFTER_DAYS: number; + EVENTS_AUTO_DELETE_ENABLED: boolean; + EVENTS_AUTO_DELETE_AFTER_DAYS: number; + LDAP_1_ENABLED: boolean; LDAP_1_USER_AUTO_CREATION: boolean; LDAP_1_HOST: string; diff --git a/packages/server/src/models/EventModel.test.ts b/packages/server/src/models/EventModel.test.ts index ae05ebf56..36306b69a 100644 --- a/packages/server/src/models/EventModel.test.ts +++ b/packages/server/src/models/EventModel.test.ts @@ -1,6 +1,6 @@ import { EventType } from '../services/database/types'; import { beforeAllDb, afterAllTests, beforeEachDb, models } from '../utils/testing/testUtils'; -import { msleep } from '../utils/time'; +import { msleep, Week } from '../utils/time'; describe('EventModel', () => { @@ -37,4 +37,52 @@ describe('EventModel', () => { expect(latest.id).toBe(allEvents[1].id); }); + test('deletion should work when there are no events', async () => { + const allEvents = (await models().event().all()); + expect(allEvents.length).toBe(0); + + await models().event().deleteOldEvents(Week); + + const remainingEvents = (await models().event().all()); + expect(remainingEvents.length).toBe(0); + }); + + test('should not delete recent events', async () => { + await models().event().create(EventType.TaskStarted, 'deleteExpiredTokens'); + + const allEvents = (await models().event().all()); + expect(allEvents.length).toBe(1); + + await models().event().deleteOldEvents(Week); + + const remainingEvents = (await models().event().all()); + expect(remainingEvents.length).toBe(1); + }); + + test('should delete events older than specified interval', async () => { + const now = Date.now(); + const aWeekAgo = now - Week; + jest.useFakeTimers(); + + for (const difference of [-10, -5, 0, 5, 10]) { + jest.setSystemTime(aWeekAgo + difference); + await models().event().create(EventType.TaskStarted, 'deleteExpiredTokens'); + } + + const allEvents = (await models().event().all()); + expect(allEvents.length).toBe(5); + + jest.setSystemTime(now); + await models().event().deleteOldEvents(Week); + + const remainingEvents = (await models().event().all()); + expect(remainingEvents.length).toBe(3); + + for (const event of remainingEvents) { + expect(event.created_time).toBeGreaterThanOrEqual(aWeekAgo); + } + + jest.useRealTimers(); + }); + }); diff --git a/packages/server/src/models/EventModel.ts b/packages/server/src/models/EventModel.ts index 6abb2fdd1..caf2729b4 100644 --- a/packages/server/src/models/EventModel.ts +++ b/packages/server/src/models/EventModel.ts @@ -33,4 +33,11 @@ export default class EventModel extends BaseModel { .first(); } + public async deleteOldEvents(ttl: number): Promise { + const cutOffDate = Date.now() - ttl; + await this.db(this.tableName) + .where('created_time', '<', cutOffDate) + .delete(); + } + } diff --git a/packages/server/src/services/TaskService.ts b/packages/server/src/services/TaskService.ts index 78a05a8ec..86559d578 100644 --- a/packages/server/src/services/TaskService.ts +++ b/packages/server/src/services/TaskService.ts @@ -32,6 +32,7 @@ export const taskIdToLabel = (taskId: TaskId): string => { [TaskId.ProcessShares]: 'Process shared items', [TaskId.ProcessEmails]: 'Process emails', [TaskId.LogHeartbeatMessage]: 'Log heartbeat message', + [TaskId.DeleteOldEvents]: 'Delete old events', }; const s = strings[taskId]; diff --git a/packages/server/src/services/database/types.ts b/packages/server/src/services/database/types.ts index 735cbc065..9448d7eca 100644 --- a/packages/server/src/services/database/types.ts +++ b/packages/server/src/services/database/types.ts @@ -136,6 +136,7 @@ export enum TaskId { ProcessShares, ProcessEmails, LogHeartbeatMessage, + DeleteOldEvents, } // AUTO-GENERATED-TYPES diff --git a/packages/server/src/utils/setupTaskService.ts b/packages/server/src/utils/setupTaskService.ts index 05d8afe8d..1a973f362 100644 --- a/packages/server/src/utils/setupTaskService.ts +++ b/packages/server/src/utils/setupTaskService.ts @@ -4,6 +4,7 @@ import TaskService, { Task, taskIdToLabel } from '../services/TaskService'; import { Services } from '../services/types'; import { logHeartbeat as logHeartbeatMessage } from './metrics'; import { Config, Env } from './types'; +import { Day } from './time'; export default async function(env: Env, models: Models, config: Config, services: Services): Promise { const taskService = new TaskService(env, models, config, services); @@ -93,6 +94,15 @@ export default async function(env: Env, models: Models, config: Config, services }); } + if (config.EVENTS_AUTO_DELETE_ENABLED) { + tasks.push({ + id: TaskId.DeleteOldEvents, + description: taskIdToLabel(TaskId.DeleteOldEvents), + schedule: '0 0 * * *', + run: (models: Models) => models.event().deleteOldEvents(config.EVENTS_AUTO_DELETE_AFTER_DAYS * Day), + }); + } + if (config.isJoplinCloud) { tasks = tasks.concat([ {