Server: Add task to automate deletion of disabled accounts

packages/server/src/config.ts

@@ -106,6 +106,7 @@ export async function initConfig(envType: Env, env: EnvVariables, overrides: any
 	const supportEmail = env.SUPPORT_EMAIL;
 
 	config_ = {
+		...env,
 		appVersion: packageJson.version,
 		appName,
 		isJoplinCloud: apiBaseUrl.includes('.joplincloud.com') || apiBaseUrl.includes('.joplincloud.local'),

packages/server/src/env.ts

@@ -89,6 +89,13 @@ const defaultEnvValues: EnvVariables = {
 
 	STRIPE_SECRET_KEY: '',
 	STRIPE_WEBHOOK_SECRET: '',
+
+	// ==================================================
+	// User data deletion
+	// ==================================================
+
+	USER_DATA_AUTO_DELETE_ENABLED: false,
+	USER_DATA_AUTO_DELETE_AFTER_DAYS: 90,
 };
 
 export interface EnvVariables {
@@ -138,6 +145,9 @@ export interface EnvVariables {
 
 	STRIPE_SECRET_KEY: string;
 	STRIPE_WEBHOOK_SECRET: string;
+
+	USER_DATA_AUTO_DELETE_ENABLED: boolean;
+	USER_DATA_AUTO_DELETE_AFTER_DAYS: number;
 }
 
 const parseBoolean = (s: string): boolean => {

packages/server/src/models/UserDeletionModel.test.ts

@@ -1,4 +1,5 @@
 import { beforeAllDb, afterAllTests, beforeEachDb, models, createUser, expectThrow } from '../utils/testing/testUtils';
+import { Day } from '../utils/time';
 
 describe('UserDeletionModel', function() {
 
@@ -143,4 +144,39 @@ describe('UserDeletionModel', function() {
 		jest.useRealTimers();
 	});
 
+	test('should auto-add users for deletion', async function() {
+		jest.useFakeTimers('modern');
+
+		const t0 = new Date('2022-02-22').getTime();
+		jest.setSystemTime(t0);
+
+		await createUser(1);
+		const user2 = await createUser(2);
+
+		await models().user().save({
+			id: user2.id,
+			enabled: 0,
+			disabled_time: t0,
+		});
+
+		await models().userDeletion().autoAdd(10, 90 * Day, 3 * Day);
+
+		expect(await models().userDeletion().count()).toBe(0);
+
+		const t1 = new Date('2022-05-30').getTime();
+		jest.setSystemTime(t1);
+
+		await models().userDeletion().autoAdd(10, 90 * Day, 3 * Day);
+
+		expect(await models().userDeletion().count()).toBe(1);
+		const d = (await models().userDeletion().all())[0];
+		expect(d.user_id).toBe(user2.id);
+
+		// Shouldn't add it again if running autoAdd() again
+		await models().userDeletion().autoAdd(10, 90 * Day, 3 * Day);
+		expect(await models().userDeletion().count()).toBe(1);
+
+		jest.useRealTimers();
+	});
+
 });

packages/server/src/models/UserDeletionModel.ts

@@ -1,4 +1,4 @@
-import { UserDeletion, Uuid } from '../services/database/types';
+import { User, UserDeletion, Uuid } from '../services/database/types';
 import { errorToString } from '../utils/errors';
 import BaseModel from './BaseModel';
 
@@ -7,6 +7,14 @@ export interface AddOptions {
 	processAccount?: boolean;
 }
 
+const defaultAddOptions = () => {
+	const d: AddOptions = {
+		processAccount: true,
+		processData: true,
+	};
+	return d;
+};
+
 export default class UserDeletionModel extends BaseModel<UserDeletion> {
 
 	protected get tableName(): string {
@@ -28,8 +36,7 @@ export default class UserDeletionModel extends BaseModel<UserDeletion> {
 
 	public async add(userId: Uuid, scheduledTime: number, options: AddOptions = null): Promise<UserDeletion> {
 		options = {
-			processAccount: true,
-			processData: true,
+			...defaultAddOptions(),
 			...options,
 		};
 
@@ -91,4 +98,26 @@ export default class UserDeletionModel extends BaseModel<UserDeletion> {
 			.where('id', deletionId);
 	}
 
+	public async autoAdd(maxAutoAddedAccounts: number, ttl: number, scheduledTime: number, options: AddOptions = null): Promise<Uuid[]> {
+		const cutOffTime = Date.now() - ttl;
+
+		const disabledUsers: User[] = await this.db('users')
+			.select(['users.id'])
+			.leftJoin('user_deletions', 'users.id', 'user_deletions.user_id')
+			.where('users.enabled', '=', 0)
+			.where('users.disabled_time', '<', cutOffTime)
+			.whereNull('user_deletions.user_id') // Only add users not already in the user_deletions table
+			.limit(maxAutoAddedAccounts);
+
+		const userIds = disabledUsers.map(d => d.id);
+
+		await this.withTransaction(async () => {
+			for (const userId of userIds) {
+				await this.add(userId, scheduledTime, options);
+			}
+		}, 'UserDeletionModel::autoAdd');
+
+		return userIds;
+	}
+
 }

packages/server/src/services/TaskService.ts

@@ -4,6 +4,7 @@ import { Config, Env } from '../utils/types';
 import BaseService from './BaseService';
 import { Event, EventType } from './database/types';
 import { Services } from './types';
+import { _ } from '@joplin/lib/locale';
 const cron = require('node-cron');
 
 const logger = Logger.create('TaskService');
@@ -17,6 +18,7 @@ export enum TaskId {
 	DeleteExpiredSessions = 6,
 	CompressOldChanges = 7,
 	ProcessUserDeletions = 8,
+	AutoAddDisabledAccountsForDeletion = 9,
 }
 
 export enum RunType {
@@ -24,6 +26,25 @@ export enum RunType {
 	Manual = 2,
 }
 
+export const taskIdToLabel = (taskId: TaskId): string => {
+	const strings: Record<TaskId, string> = {
+		[TaskId.DeleteExpiredTokens]: _('Delete expired tokens'),
+		[TaskId.UpdateTotalSizes]: _('Update total sizes'),
+		[TaskId.HandleOversizedAccounts]: _('Process oversized accounts'),
+		[TaskId.HandleBetaUserEmails]: 'Process beta user emails',
+		[TaskId.HandleFailedPaymentSubscriptions]: _('Process failed payment subscriptions'),
+		[TaskId.DeleteExpiredSessions]: _('Delete expired sessions'),
+		[TaskId.CompressOldChanges]: _('Compress old changes'),
+		[TaskId.ProcessUserDeletions]: _('Process user deletions'),
+		[TaskId.AutoAddDisabledAccountsForDeletion]: _('Auto-add disabled accounts for deletion'),
+	};
+
+	const s = strings[taskId];
+	if (!s) throw new Error(`No such task: ${taskId}`);
+
+	return s;
+};
+
 const runTypeToString = (runType: RunType) => {
 	if (runType === RunType.Scheduled) return 'scheduled';
 	if (runType === RunType.Manual) return 'manual';

packages/server/src/services/UserDeletionService.ts

@@ -1,8 +1,8 @@
 import Logger from '@joplin/lib/Logger';
 import { Pagination } from '../models/utils/pagination';
-import { msleep } from '../utils/time';
+import { Day, msleep } from '../utils/time';
 import BaseService from './BaseService';
-import { UserDeletion, UserFlagType, Uuid } from './database/types';
+import { BackupItemType, UserDeletion, UserFlagType, Uuid } from './database/types';
 
 const logger = Logger.create('UserDeletionService');
 
@@ -59,6 +59,21 @@ export default class UserDeletionService extends BaseService {
 	private async deleteUserAccount(userId: Uuid, _options: DeletionJobOptions = null) {
 		logger.info(`Deleting user account: ${userId}`);
 
+		const user = await this.models.user().load(userId);
+		if (!user) throw new Error(`No such user: ${userId}`);
+
+		const flags = await this.models.userFlag().allByUserId(userId);
+
+		await this.models.backupItem().add(
+			BackupItemType.UserAccount,
+			user.email,
+			JSON.stringify({
+				user,
+				flags,
+			}),
+			userId
+		);
+
 		await this.models.userFlag().add(userId, UserFlagType.UserDeletionInProgress);
 
 		await this.models.session().deleteByUserId(userId);
@@ -93,6 +108,24 @@ export default class UserDeletionService extends BaseService {
 		logger.info('Completed user deletion: ', deletion.id);
 	}
 
+	public async autoAddForDeletion() {
+		const addedUserIds = await this.models.userDeletion().autoAdd(
+			10,
+			this.config.USER_DATA_AUTO_DELETE_AFTER_DAYS * Day,
+			3 * Day,
+			{
+				processAccount: true,
+				processData: true,
+			}
+		);
+
+		if (addedUserIds.length) {
+			logger.info(`autoAddForDeletion: Queued ${addedUserIds.length} users for deletions: ${addedUserIds.join(', ')}`);
+		} else {
+			logger.info('autoAddForDeletion: No users were queued for deletion');
+		}
+	}
+
 	public async processNextDeletionJob() {
 		const deletion = await this.models.userDeletion().next();
 		if (!deletion) return;

packages/server/src/utils/setupTaskService.ts

@@ -1,5 +1,5 @@
 import { Models } from '../models/factory';
-import TaskService, { Task, TaskId } from '../services/TaskService';
+import TaskService, { Task, TaskId, taskIdToLabel } from '../services/TaskService';
 import { Services } from '../services/types';
 import { Config, Env } from './types';
 
@@ -9,28 +9,28 @@ export default function(env: Env, models: Models, config: Config, services: Serv
 	let tasks: Task[] = [
 		{
 			id: TaskId.DeleteExpiredTokens,
-			description: 'Delete expired tokens',
+			description: taskIdToLabel(TaskId.DeleteExpiredTokens),
 			schedule: '0 */6 * * *',
 			run: (models: Models) => models.token().deleteExpiredTokens(),
 		},
 
 		{
 			id: TaskId.UpdateTotalSizes,
-			description: 'Update total sizes',
+			description: taskIdToLabel(TaskId.UpdateTotalSizes),
 			schedule: '0 * * * *',
 			run: (models: Models) => models.item().updateTotalSizes(),
 		},
 
 		{
 			id: TaskId.CompressOldChanges,
-			description: 'Compress old changes',
+			description: taskIdToLabel(TaskId.CompressOldChanges),
 			schedule: '0 0 */2 * *',
 			run: (models: Models) => models.change().compressOldChanges(),
 		},
 
 		{
 			id: TaskId.ProcessUserDeletions,
-			description: 'Process user deletions',
+			description: taskIdToLabel(TaskId.ProcessUserDeletions),
 			schedule: '0 */6 * * *',
 			run: (_models: Models, services: Services) => services.userDeletion.runMaintenance(),
 		},
@@ -41,30 +41,39 @@ export default function(env: Env, models: Models, config: Config, services: Serv
 		// the UpdateTotalSizes task being run.
 		{
 			id: TaskId.HandleOversizedAccounts,
-			description: 'Process oversized accounts',
+			description: taskIdToLabel(TaskId.HandleOversizedAccounts),
 			schedule: '30 */2 * * *',
 			run: (models: Models) => models.user().handleOversizedAccounts(),
 		},
 
 		// {
 		// 	id: TaskId.DeleteExpiredSessions,
-		// 	description: 'Delete expired sessions',
+		// 	description: taskIdToLabel(TaskId.DeleteExpiredSessions),
 		// 	schedule: '0 */6 * * *',
 		// 	run: (models: Models) => models.session().deleteExpiredSessions(),
 		// },
 	];
 
+	if (config.USER_DATA_AUTO_DELETE_ENABLED) {
+		tasks.push({
+			id: TaskId.AutoAddDisabledAccountsForDeletion,
+			description: taskIdToLabel(TaskId.AutoAddDisabledAccountsForDeletion),
+			schedule: '0 14 * * *',
+			run: (_models: Models, services: Services) => services.userDeletion.autoAddForDeletion(),
+		});
+	}
+
 	if (config.isJoplinCloud) {
 		tasks = tasks.concat([
 			{
 				id: TaskId.HandleBetaUserEmails,
-				description: 'Process beta user emails',
+				description: taskIdToLabel(TaskId.HandleBetaUserEmails),
 				schedule: '0 12 * * *',
 				run: (models: Models) => models.user().handleBetaUserEmails(),
 			},
 			{
 				id: TaskId.HandleFailedPaymentSubscriptions,
-				description: 'Process failed payment subscriptions',
+				description: taskIdToLabel(TaskId.HandleFailedPaymentSubscriptions),
 				schedule: '0 13 * * *',
 				run: (models: Models) => models.user().handleFailedPaymentSubscriptions(),
 			},

packages/server/src/utils/types.ts

@@ -7,7 +7,7 @@ import { Account } from '../models/UserModel';
 import { Services } from '../services/types';
 import { Routers } from './routeUtils';
 import { DbConnection } from '../db';
-import { MailerSecurity } from '../env';
+import { EnvVariables, MailerSecurity } from '../env';
 
 export enum Env {
 	Dev = 'dev',
@@ -130,7 +130,7 @@ export interface StorageDriverConfig {
 	bucket?: string;
 }
 
-export interface Config {
+export interface Config extends EnvVariables {
 	appVersion: string;
 	appName: string;
 	env: Env;