From 75f729620e19a8e639daf77bd416f8eac4dcfd66 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Tue, 23 Nov 2021 16:25:36 +0000 Subject: [PATCH] Server: Added task to compress changes older than 6 months --- ...hangesCommand.ts => CompressOldChangesCommand.ts} | 8 ++++---- packages/server/src/models/ChangeModel.test.ts | 8 ++++---- packages/server/src/models/ChangeModel.ts | 12 +++++++----- packages/server/src/services/TaskService.ts | 1 + packages/server/src/utils/setupCommands.ts | 4 ++-- packages/server/src/utils/setupTaskService.ts | 8 ++++++++ 6 files changed, 26 insertions(+), 15 deletions(-) rename packages/server/src/commands/{DeleteOldChangesCommand.ts => CompressOldChangesCommand.ts} (62%) diff --git a/packages/server/src/commands/DeleteOldChangesCommand.ts b/packages/server/src/commands/CompressOldChangesCommand.ts similarity index 62% rename from packages/server/src/commands/DeleteOldChangesCommand.ts rename to packages/server/src/commands/CompressOldChangesCommand.ts index 87ceae925..29158e0d7 100644 --- a/packages/server/src/commands/DeleteOldChangesCommand.ts +++ b/packages/server/src/commands/CompressOldChangesCommand.ts @@ -6,14 +6,14 @@ interface Argv { ttl: number; } -export default class DeleteOldChangesCommand extends BaseCommand { +export default class CompressOldChangesCommand extends BaseCommand { public command() { - return 'delete-old-changes'; + return 'compress-old-changes'; } public description() { - return 'deletes old changes'; + return 'compresses old changes by discarding consecutive updates'; } public options(): Record { @@ -26,7 +26,7 @@ export default class DeleteOldChangesCommand extends BaseCommand { } public async run(argv: Argv, runContext: RunContext): Promise { - await runContext.models.change().deleteOldChanges(argv.ttl ? argv.ttl * Day : null); + await runContext.models.change().compressOldChanges(argv.ttl ? argv.ttl * Day : null); } } diff --git a/packages/server/src/models/ChangeModel.test.ts b/packages/server/src/models/ChangeModel.test.ts index b8a6ed879..02ddf7b5b 100644 --- a/packages/server/src/models/ChangeModel.test.ts +++ b/packages/server/src/models/ChangeModel.test.ts @@ -227,13 +227,13 @@ describe('ChangeModel', function() { expect(await models().change().count()).toBe(7); // Shouldn't do anything initially because it only deletes old changes. - await models().change().deleteOldChanges(); + await models().change().compressOldChanges(); expect(await models().change().count()).toBe(7); // 180 days after T4, it should delete all U1 updates events except for // the last one jest.setSystemTime(new Date(t4 + changeTtl).getTime()); - await models().change().deleteOldChanges(); + await models().change().compressOldChanges(); expect(await models().change().count()).toBe(5); { const updateChange = (await models().change().all()).find(c => c.item_id === note1.id && c.type === ChangeType.Update); @@ -247,13 +247,13 @@ describe('ChangeModel', function() { // there's only one note 2 change that is older than 90 days at this // point. jest.setSystemTime(new Date(t5 + changeTtl).getTime()); - await models().change().deleteOldChanges(); + await models().change().compressOldChanges(); expect(await models().change().count()).toBe(5); // After T6, more than 90 days later - now the change at T5 should be // deleted, keeping only the change at T6. jest.setSystemTime(new Date(t6 + changeTtl).getTime()); - await models().change().deleteOldChanges(); + await models().change().compressOldChanges(); expect(await models().change().count()).toBe(4); { const updateChange = (await models().change().all()).find(c => c.item_id === note2.id && c.type === ChangeType.Update); diff --git a/packages/server/src/models/ChangeModel.ts b/packages/server/src/models/ChangeModel.ts index 0e66bffe6..3388cc683 100644 --- a/packages/server/src/models/ChangeModel.ts +++ b/packages/server/src/models/ChangeModel.ts @@ -309,7 +309,9 @@ export default class ChangeModel extends BaseModel { return output; } - public async deleteOldChanges(ttl: number = null) { + // See spec for complete documentation: + // https://joplinapp.org/spec/server_delta_sync/#regarding-the-deletion-of-old-change-events + public async compressOldChanges(ttl: number = null) { ttl = ttl === null ? defaultChangeTtl : ttl; const cutOffDate = Date.now() - ttl; const limit = 1000; @@ -324,7 +326,7 @@ export default class ChangeModel extends BaseModel { let error: Error = null; let totalDeletedCount = 0; - logger.info(`deleteOldChanges: Processing changes older than: ${formatDateTime(cutOffDate)} (${cutOffDate})`); + logger.info(`compressOldChanges: Processing changes older than: ${formatDateTime(cutOffDate)} (${cutOffDate})`); while (true) { // First get all the UPDATE changes before the specified date, and @@ -373,14 +375,14 @@ export default class ChangeModel extends BaseModel { totalDeletedCount += deletedCount; doneItemIds.push(row.item_id); } - }, 'ChangeModel::deleteOldChanges'); + }, 'ChangeModel::compressOldChanges'); - logger.info(`deleteOldChanges: Processed: ${doneItemIds.length} items. Deleted: ${totalDeletedCount} changes.`); + logger.info(`compressOldChanges: Processed: ${doneItemIds.length} items. Deleted: ${totalDeletedCount} changes.`); if (error) throw error; } - logger.info(`deleteOldChanges: Finished processing. Done ${doneItemIds.length} items. Deleted: ${totalDeletedCount} changes.`); + logger.info(`compressOldChanges: Finished processing. Done ${doneItemIds.length} items. Deleted: ${totalDeletedCount} changes.`); } public async save(change: Change, options: SaveOptions = {}): Promise { diff --git a/packages/server/src/services/TaskService.ts b/packages/server/src/services/TaskService.ts index d224ba95d..c9c8ebaa5 100644 --- a/packages/server/src/services/TaskService.ts +++ b/packages/server/src/services/TaskService.ts @@ -13,6 +13,7 @@ export enum TaskId { HandleBetaUserEmails = 4, HandleFailedPaymentSubscriptions = 5, DeleteExpiredSessions = 6, + CompressOldChanges = 7, } export enum RunType { diff --git a/packages/server/src/utils/setupCommands.ts b/packages/server/src/utils/setupCommands.ts index 2ff7ecc1a..dada621f3 100644 --- a/packages/server/src/utils/setupCommands.ts +++ b/packages/server/src/utils/setupCommands.ts @@ -1,7 +1,7 @@ import yargs = require('yargs'); import BaseCommand from '../commands/BaseCommand'; import DbCommand from '../commands/DbCommand'; -import DeleteOldChangesCommand from '../commands/DeleteOldChangesCommand'; +import CompressOldChangesCommand from '../commands/CompressOldChangesCommand'; import StorageCommand from '../commands/StorageCommand'; import MigrateCommand from '../commands/MigrateCommand'; @@ -16,7 +16,7 @@ export default async function setupCommands(): Promise { const commands: BaseCommand[] = [ new MigrateCommand(), new DbCommand(), - new DeleteOldChangesCommand(), + new CompressOldChangesCommand(), new StorageCommand(), ]; diff --git a/packages/server/src/utils/setupTaskService.ts b/packages/server/src/utils/setupTaskService.ts index e9c22bff3..4e677cbc0 100644 --- a/packages/server/src/utils/setupTaskService.ts +++ b/packages/server/src/utils/setupTaskService.ts @@ -12,6 +12,7 @@ export default function(env: Env, models: Models, config: Config): TaskService { schedule: '0 */6 * * *', run: (models: Models) => models.token().deleteExpiredTokens(), }, + { id: TaskId.UpdateTotalSizes, description: 'Update total sizes', @@ -19,6 +20,13 @@ export default function(env: Env, models: Models, config: Config): TaskService { run: (models: Models) => models.item().updateTotalSizes(), }, + { + id: TaskId.CompressOldChanges, + description: 'Compress old changes', + schedule: '0 0 */2 * *', + run: (models: Models) => models.change().compressOldChanges(), + }, + // Need to do it relatively frequently so that if the user fixes // whatever was causing the oversized account, they can get it // re-enabled quickly. Also it's done on minute 30 because it depends on