Server: Added task to compress changes older than 6 months

packages/server/src/commands/CompressOldChangesCommand.ts

@@ -6,14 +6,14 @@ interface Argv {
 	ttl: number;
 }
 
-export default class DeleteOldChangesCommand extends BaseCommand {
+export default class CompressOldChangesCommand extends BaseCommand {
 
 	public command() {
-		return 'delete-old-changes';
+		return 'compress-old-changes';
 	}
 
 	public description() {
-		return 'deletes old changes';
+		return 'compresses old changes by discarding consecutive updates';
 	}
 
 	public options(): Record<string, Options> {
@@ -26,7 +26,7 @@ export default class DeleteOldChangesCommand extends BaseCommand {
 	}
 
 	public async run(argv: Argv, runContext: RunContext): Promise<void> {
-		await runContext.models.change().deleteOldChanges(argv.ttl ? argv.ttl * Day : null);
+		await runContext.models.change().compressOldChanges(argv.ttl ? argv.ttl * Day : null);
 	}
 
 }

packages/server/src/models/ChangeModel.test.ts

@@ -227,13 +227,13 @@ describe('ChangeModel', function() {
 		expect(await models().change().count()).toBe(7);
 
 		// Shouldn't do anything initially because it only deletes old changes.
-		await models().change().deleteOldChanges();
+		await models().change().compressOldChanges();
 		expect(await models().change().count()).toBe(7);
 
 		// 180 days after T4, it should delete all U1 updates events except for
 		// the last one
 		jest.setSystemTime(new Date(t4 + changeTtl).getTime());
-		await models().change().deleteOldChanges();
+		await models().change().compressOldChanges();
 		expect(await models().change().count()).toBe(5);
 		{
 			const updateChange = (await models().change().all()).find(c => c.item_id === note1.id && c.type === ChangeType.Update);
@@ -247,13 +247,13 @@ describe('ChangeModel', function() {
 		// there's only one note 2 change that is older than 90 days at this
 		// point.
 		jest.setSystemTime(new Date(t5 + changeTtl).getTime());
-		await models().change().deleteOldChanges();
+		await models().change().compressOldChanges();
 		expect(await models().change().count()).toBe(5);
 
 		// After T6, more than 90 days later - now the change at T5 should be
 		// deleted, keeping only the change at T6.
 		jest.setSystemTime(new Date(t6 + changeTtl).getTime());
-		await models().change().deleteOldChanges();
+		await models().change().compressOldChanges();
 		expect(await models().change().count()).toBe(4);
 		{
 			const updateChange = (await models().change().all()).find(c => c.item_id === note2.id && c.type === ChangeType.Update);

packages/server/src/models/ChangeModel.ts

@@ -309,7 +309,9 @@ export default class ChangeModel extends BaseModel<Change> {
 		return output;
 	}
 
-	public async deleteOldChanges(ttl: number = null) {
+	// See spec for complete documentation:
+	// https://joplinapp.org/spec/server_delta_sync/#regarding-the-deletion-of-old-change-events
+	public async compressOldChanges(ttl: number = null) {
 		ttl = ttl === null ? defaultChangeTtl : ttl;
 		const cutOffDate = Date.now() - ttl;
 		const limit = 1000;
@@ -324,7 +326,7 @@ export default class ChangeModel extends BaseModel<Change> {
 		let error: Error = null;
 		let totalDeletedCount = 0;
 
-		logger.info(`deleteOldChanges: Processing changes older than: ${formatDateTime(cutOffDate)} (${cutOffDate})`);
+		logger.info(`compressOldChanges: Processing changes older than: ${formatDateTime(cutOffDate)} (${cutOffDate})`);
 
 		while (true) {
 			// First get all the UPDATE changes before the specified date, and
@@ -373,14 +375,14 @@ export default class ChangeModel extends BaseModel<Change> {
 					totalDeletedCount += deletedCount;
 					doneItemIds.push(row.item_id);
 				}
-			}, 'ChangeModel::deleteOldChanges');
+			}, 'ChangeModel::compressOldChanges');
 
-			logger.info(`deleteOldChanges: Processed: ${doneItemIds.length} items. Deleted: ${totalDeletedCount} changes.`);
+			logger.info(`compressOldChanges: Processed: ${doneItemIds.length} items. Deleted: ${totalDeletedCount} changes.`);
 
 			if (error) throw error;
 		}
 
-		logger.info(`deleteOldChanges: Finished processing. Done ${doneItemIds.length} items. Deleted: ${totalDeletedCount} changes.`);
+		logger.info(`compressOldChanges: Finished processing. Done ${doneItemIds.length} items. Deleted: ${totalDeletedCount} changes.`);
 	}
 
 	public async save(change: Change, options: SaveOptions = {}): Promise<Change> {

packages/server/src/services/TaskService.ts

@@ -13,6 +13,7 @@ export enum TaskId {
 	HandleBetaUserEmails = 4,
 	HandleFailedPaymentSubscriptions = 5,
 	DeleteExpiredSessions = 6,
+	CompressOldChanges = 7,
 }
 
 export enum RunType {

packages/server/src/utils/setupCommands.ts

@@ -1,7 +1,7 @@
 import yargs = require('yargs');
 import BaseCommand from '../commands/BaseCommand';
 import DbCommand from '../commands/DbCommand';
-import DeleteOldChangesCommand from '../commands/DeleteOldChangesCommand';
+import CompressOldChangesCommand from '../commands/CompressOldChangesCommand';
 import StorageCommand from '../commands/StorageCommand';
 import MigrateCommand from '../commands/MigrateCommand';
 
@@ -16,7 +16,7 @@ export default async function setupCommands(): Promise<Commands> {
 	const commands: BaseCommand[] = [
 		new MigrateCommand(),
 		new DbCommand(),
-		new DeleteOldChangesCommand(),
+		new CompressOldChangesCommand(),
 		new StorageCommand(),
 	];
 

packages/server/src/utils/setupTaskService.ts

@@ -12,6 +12,7 @@ export default function(env: Env, models: Models, config: Config): TaskService {
 			schedule: '0 */6 * * *',
 			run: (models: Models) => models.token().deleteExpiredTokens(),
 		},
+
 		{
 			id: TaskId.UpdateTotalSizes,
 			description: 'Update total sizes',
@@ -19,6 +20,13 @@ export default function(env: Env, models: Models, config: Config): TaskService {
 			run: (models: Models) => models.item().updateTotalSizes(),
 		},
 
+		{
+			id: TaskId.CompressOldChanges,
+			description: 'Compress old changes',
+			schedule: '0 0 */2 * *',
+			run: (models: Models) => models.change().compressOldChanges(),
+		},
+
 		// Need to do it relatively frequently so that if the user fixes
 		// whatever was causing the oversized account, they can get it
 		// re-enabled quickly. Also it's done on minute 30 because it depends on