tests

.eslintignore

@@ -76,6 +76,9 @@ packages/app-cli/app/command-e2ee.js.map
 packages/app-cli/app/command-settingschema.d.ts
 packages/app-cli/app/command-settingschema.js
 packages/app-cli/app/command-settingschema.js.map
+packages/app-cli/app/command-testing.d.ts
+packages/app-cli/app/command-testing.js
+packages/app-cli/app/command-testing.js.map
 packages/app-cli/app/services/plugins/PluginRunner.d.ts
 packages/app-cli/app/services/plugins/PluginRunner.js
 packages/app-cli/app/services/plugins/PluginRunner.js.map
@@ -109,6 +112,9 @@ packages/app-cli/tests/services/plugins/sandboxProxy.js.map
 packages/app-cli/tests/testUtils.d.ts
 packages/app-cli/tests/testUtils.js
 packages/app-cli/tests/testUtils.js.map
+packages/app-cli/tools/populateDatabase.d.ts
+packages/app-cli/tools/populateDatabase.js
+packages/app-cli/tools/populateDatabase.js.map
 packages/app-desktop/ElectronAppWrapper.d.ts
 packages/app-desktop/ElectronAppWrapper.js
 packages/app-desktop/ElectronAppWrapper.js.map

.gitignore

@@ -61,6 +61,9 @@ packages/app-cli/app/command-e2ee.js.map
 packages/app-cli/app/command-settingschema.d.ts
 packages/app-cli/app/command-settingschema.js
 packages/app-cli/app/command-settingschema.js.map
+packages/app-cli/app/command-testing.d.ts
+packages/app-cli/app/command-testing.js
+packages/app-cli/app/command-testing.js.map
 packages/app-cli/app/services/plugins/PluginRunner.d.ts
 packages/app-cli/app/services/plugins/PluginRunner.js
 packages/app-cli/app/services/plugins/PluginRunner.js.map
@@ -94,6 +97,9 @@ packages/app-cli/tests/services/plugins/sandboxProxy.js.map
 packages/app-cli/tests/testUtils.d.ts
 packages/app-cli/tests/testUtils.js
 packages/app-cli/tests/testUtils.js.map
+packages/app-cli/tools/populateDatabase.d.ts
+packages/app-cli/tools/populateDatabase.js
+packages/app-cli/tools/populateDatabase.js.map
 packages/app-desktop/ElectronAppWrapper.d.ts
 packages/app-desktop/ElectronAppWrapper.js
 packages/app-desktop/ElectronAppWrapper.js.map

packages/app-cli/.gitignore

@@ -23,4 +23,6 @@ tests/support/dropbox-auth.txt
 tests/support/nextcloud-auth.json
 tests/support/onedrive-auth.txt
 build/
-patches/
+patches/
+createUsers-*.txt
+tools/temp/

packages/app-cli/app/cli-utils.js

@@ -89,7 +89,7 @@ cliUtils.makeCommandArgs = function(cmd, argv) {
 		flags = cliUtils.parseFlags(flags);
 
 		if (!flags.arg) {
-			booleanFlags.push(flags.short);
+			if (flags.short) booleanFlags.push(flags.short);
 			if (flags.long) booleanFlags.push(flags.long);
 		}
 

packages/app-cli/app/command-testing.ts

@@ -0,0 +1,95 @@
+const { BaseCommand } = require('./base-command.js');
+import { reg } from '@joplin/lib/registry';
+import Note from '@joplin/lib/models/Note';
+import uuid from '@joplin/lib/uuid';
+import populateDatabase from '@joplin/lib/services/debug/populateDatabase';
+
+function randomElement(array: any[]): any {
+	if (!array.length) return null;
+	return array[Math.floor(Math.random() * array.length)];
+}
+
+function itemCount(args: any) {
+	const count = Number(args.arg0);
+	if (!count || isNaN(count)) throw new Error('Note count must be specified');
+	return count;
+}
+
+class Command extends BaseCommand {
+	usage() {
+		return 'testing <command> [arg0]';
+	}
+
+	description() {
+		return 'testing';
+	}
+
+	enabled() {
+		return false;
+	}
+
+	options(): any[] {
+		return [
+			['--folder-count <count>', 'Folders to create'],
+			['--note-count <count>', 'Notes to create'],
+			['--tag-count <count>', 'Tags to create'],
+			['--tags-per-note <count>', 'Tags per note'],
+			['--silent', 'Silent'],
+		];
+	}
+
+	async action(args: any) {
+		const { command, options } = args;
+
+		if (command === 'populate') {
+			await populateDatabase(reg.db(), {
+				folderCount: options['folder-count'],
+				noteCount: options['note-count'],
+				tagCount: options['tag-count'],
+				tagsPerNote: options['tags-per-note'],
+				silent: options['silent'],
+			});
+		}
+
+		const promises: any[] = [];
+
+		if (command === 'createRandomNotes') {
+			const noteCount = itemCount(args);
+
+			for (let i = 0; i < noteCount; i++) {
+				promises.push(Note.save({
+					title: `Note ${uuid.createNano()}`,
+				}));
+			}
+		}
+
+		if (command === 'updateRandomNotes') {
+			const noteCount = itemCount(args);
+
+			const noteIds = await Note.allIds();
+
+			for (let i = 0; i < noteCount; i++) {
+				const noteId = randomElement(noteIds);
+				promises.push(Note.save({
+					id: noteId,
+					title: `Note ${uuid.createNano()}`,
+				}));
+			}
+		}
+
+		if (command === 'deleteRandomNotes') {
+			const noteCount = itemCount(args);
+			const noteIds = await Note.allIds();
+
+			for (let i = 0; i < noteCount; i++) {
+				const noteId = randomElement(noteIds);
+				promises.push(Note.delete(noteId));
+			}
+		}
+
+		await Promise.all(promises);
+	}
+
+}
+
+module.exports = Command;

packages/app-cli/createUsers.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Start the server with:
+#
+# JOPLIN_IS_TESTING=1 npm run start-dev
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+# curl --data '{"action": "clearDatabase"}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
+
+# SMALL
+
+# curl --data '{"action": "createTestUsers", "count": 400, "fromNum": 1}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
+
+NUM=398
+while [ "$NUM" -lt 400 ]; do
+	NUM=$(( NUM + 1 ))
+
+	echo "User $NUM"
+
+	CMD_FILE="$SCRIPT_DIR/createUsers-$NUM.txt"
+	PROFILE_DIR=~/.config/joplindev-testing-$NUM
+	USER_EMAIL="user$NUM@example.com"
+
+	rm -rf "$CMD_FILE" "$PROFILE_DIR"
+	touch "$CMD_FILE"
+
+	FLAG_FOLDER_COUNT=100
+	FLAG_NOTE_COUNT=1000
+	FLAG_TAG_COUNT=20
+
+	if [ "$NUM" -gt 300 ]; then
+		FLAG_FOLDER_COUNT=2000
+		FLAG_NOTE_COUNT=10000
+		FLAG_TAG_COUNT=200
+	fi
+
+	if [ "$NUM" -gt 399 ]; then
+		FLAG_FOLDER_COUNT=10000
+		FLAG_NOTE_COUNT=150000
+		FLAG_TAG_COUNT=2000
+	fi
+
+	echo "testing populate --silent --folder-count $FLAG_FOLDER_COUNT --note-count $FLAG_NOTE_COUNT --tag-count $FLAG_TAG_COUNT" >> "$CMD_FILE"
+	echo "config keychain.supported 0" >> "$CMD_FILE" 
+	echo "config sync.target 10" >> "$CMD_FILE" 
+	echo "config sync.10.username $USER_EMAIL" >> "$CMD_FILE" 
+	echo "config sync.10.password hunter1hunter2hunter3" >> "$CMD_FILE"
+	echo "sync" >> "$CMD_FILE"
+
+	npm start -- --profile "$PROFILE_DIR" batch "$CMD_FILE"
+done

packages/app-cli/package.json

@@ -10,6 +10,7 @@
     "test-ci": "jest --config=jest.config.js --forceExit",
     "build": "gulp build",
     "start": "gulp build -L && node \"build/main.js\" --stack-trace-enabled --log-level debug --env dev",
+    "start-no-build": "node \"build/main.js\" --stack-trace-enabled --log-level debug --env dev",
     "tsc": "node node_modules/typescript/bin/tsc --project tsconfig.json",
     "watch": "node node_modules/typescript/bin/tsc --watch --project tsconfig.json"
   },

packages/app-cli/tools/populateDatabase.ts

@@ -0,0 +1,108 @@
+// This script can be used to simulate a running production environment, by
+// having multiple users in parallel changing notes and synchronising.
+//
+// To get it working:
+//
+// - Run the Postgres database -- `sudo docker-compose --file docker-compose.db-dev.yml up`
+// - Update the DB parameters in ~/joplin-credentials/server.env to use the dev
+//   database
+// - Run the server - `JOPLIN_IS_TESTING=1 npm run start-dev`
+// - Then run this script - `node populateDatabase.js`
+//
+// Currently it doesn't actually create the users, so that should be done using:
+//
+// curl --data '{"action": "createTestUsers", "count": 400, "fromNum": 1}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
+//
+// That will create n users with email `user<n>@example.com`
+
+import * as fs from 'fs-extra';
+import { homedir } from 'os';
+import { execCommand2 } from '@joplin/tools/tool-utils';
+import { chdir } from 'process';
+
+const minUserNum = 1;
+const maxUserNum = 400;
+
+const cliDir = `${__dirname}/..`;
+const tempDir = `${__dirname}/temp`;
+
+function randomInt(min: number, max: number) {
+	min = Math.ceil(min);
+	max = Math.floor(max);
+	return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+const processing_: Record<number, boolean> = {};
+
+const processUser = async (userNum: number) => {
+	if (processing_[userNum]) {
+		console.info(`User already being processed: ${userNum} - skipping`);
+		return;
+	}
+
+	processing_[userNum] = true;
+
+	try {
+		const userEmail = `user${userNum}@example.com`;
+		const userPassword = 'hunter1hunter2hunter3';
+		const commandFile = `${tempDir}/populateDatabase-${userNum}.txt`;
+		const profileDir = `${homedir()}/.config/joplindev-populate/joplindev-testing-${userNum}`;
+
+		const commands: string[] = [];
+		const jackpot = Math.random() >= 0.95 ? 100 : 1;
+
+		commands.push(`testing createRandomNotes ${randomInt(1, 500 * jackpot)}`);
+		commands.push(`testing updateRandomNotes ${randomInt(1, 1500 * jackpot)}`);
+		commands.push(`testing deleteRandomNotes ${randomInt(1, 200 * jackpot)}`);
+		commands.push('config keychain.supported 0');
+		commands.push('config sync.target 10');
+		commands.push(`config sync.10.username ${userEmail}`);
+		commands.push(`config sync.10.password ${userPassword}`);
+		commands.push('sync');
+
+		await fs.writeFile(commandFile, commands.join('\n'), 'utf8');
+
+		await chdir(cliDir);
+
+		await execCommand2(['npm', 'run', 'start-no-build', '--', '--profile', profileDir, 'batch', commandFile]);
+	} catch (error) {
+		console.error(`Could not process user ${userNum}:`, error);
+	} finally {
+		delete processing_[userNum];
+	}
+};
+
+const waitForProcessing = (count: number) => {
+	return new Promise((resolve) => {
+		const iid = setInterval(() => {
+			if (Object.keys(processing_).length <= count) {
+				clearInterval(iid);
+				resolve(null);
+			}
+		}, 100);
+	});
+};
+
+const main = async () => {
+	await fs.mkdirp(tempDir);
+
+	// Build the app once before starting, because we'll use start-no-build to
+	// run the scripts (faster)
+	await execCommand2(['npm', 'run', 'build']);
+
+	const focusUserNum = 400;
+
+	while (true) {
+		let userNum = randomInt(minUserNum, maxUserNum);
+
+		if (Math.random() >= .7) userNum = focusUserNum;
+
+		void processUser(userNum);
+		await waitForProcessing(10);
+	}
+};
+
+main().catch((error) => {
+	console.error('Fatal error', error);
+	process.exit(1);
+});

packages/server/src/db.ts

@@ -122,8 +122,16 @@ export function setupSlowQueryLog(connection: DbConnection, slowQueryLogMinDurat
 
 	const queryInfos: Record<any, QueryInfo> = {};
 
+	// These queries do not return a response, so "query-response" is not
+	// called.
+	const ignoredQueries = /^BEGIN|SAVEPOINT|RELEASE SAVEPOINT|COMMIT|ROLLBACK/gi;
+
 	connection.on('query', (data) => {
-		const timeoutId = makeSlowQueryHandler(slowQueryLogMinDuration, connection, data.sql, data.bindings);
+		const sql: string = data.sql;
+
+		if (!sql || sql.match(ignoredQueries)) return;
+
+		const timeoutId = makeSlowQueryHandler(slowQueryLogMinDuration, connection, sql, data.bindings);
 
 		queryInfos[data.__knexQueryUid] = {
 			timeoutId,

packages/server/src/middleware/notificationHandler.ts

@@ -42,6 +42,13 @@ async function handleSqliteInProdNotification(ctx: AppContext) {
 	}
 }
 
+function levelClassName(level: NotificationLevel): string {
+	if (level === NotificationLevel.Important) return 'is-warning';
+	if (level === NotificationLevel.Normal) return 'is-info';
+	if (level === NotificationLevel.Error) return 'is-danger';
+	throw new Error(`Unknown level: ${level}`);
+}
+
 async function makeNotificationViews(ctx: AppContext): Promise<NotificationView[]> {
 	const markdownIt = new MarkdownIt();
 
@@ -52,7 +59,7 @@ async function makeNotificationViews(ctx: AppContext): Promise<NotificationView[
 		views.push({
 			id: n.id,
 			messageHtml: markdownIt.render(n.message),
-			level: n.level === NotificationLevel.Important ? 'warning' : 'info',
+			levelClassName: levelClassName(n.level),
 			closeUrl: notificationModel.closeUrl(n.id),
 		});
 	}

packages/server/src/models/NotificationModel.ts

@@ -1,8 +1,10 @@
 import { Notification, NotificationLevel, Uuid } from '../services/database/types';
 import { ErrorUnprocessableEntity } from '../utils/errors';
+import uuidgen from '../utils/uuidgen';
 import BaseModel, { ValidateOptions } from './BaseModel';
 
 export enum NotificationKey {
+	Any = 'any',
 	ConfirmEmail = 'confirmEmail',
 	PasswordSet = 'passwordSet',
 	EmailConfirmed = 'emailConfirmed',
@@ -52,6 +54,10 @@ export default class NotificationModel extends BaseModel<Notification> {
 				level: NotificationLevel.Normal,
 				message: 'Thank you! Your account has been successfully upgraded to Pro.',
 			},
+			[NotificationKey.Any]: {
+				level: NotificationLevel.Normal,
+				message: '',
+			},
 		};
 
 		const type = notificationTypes[key];
@@ -72,7 +78,9 @@ export default class NotificationModel extends BaseModel<Notification> {
 			}
 		}
 
-		return this.save({ key, message, level, owner_id: userId });
+		const actualKey = key === NotificationKey.Any ? `any_${uuidgen()}` : key;
+
+		return this.save({ key: actualKey, message, level, owner_id: userId });
 	}
 
 	public async markAsRead(userId: Uuid, key: NotificationKey): Promise<void> {

packages/server/src/models/SubscriptionModel.ts

@@ -98,7 +98,7 @@ export default class SubscriptionModel extends BaseModel<Subscription> {
 			// failed.
 			//
 			// We don't update the user can_upload and enabled properties here
-			// because it's done after a few days from CronService.
+			// because it's done after a few days from TaskService.
 			if (!sub.last_payment_failed_time) {
 				const user = await this.models().user().load(sub.user_id, { fields: ['email', 'id', 'full_name'] });
 

packages/server/src/models/UserModel.ts

@@ -469,4 +469,12 @@ export default class UserModel extends BaseModel<User> {
 		}, 'UserModel::save');
 	}
 
+	public async saveMulti(users: User[], options: SaveOptions = {}): Promise<void> {
+		await this.withTransaction(async () => {
+			for (const user of users) {
+				await this.save(user, options);
+			}
+		}, 'UserModel::saveMulti');
+	}
+
 }

packages/server/src/routes/api/debug.ts

@@ -1,5 +1,5 @@
 import config from '../../config';
-import { createTestUsers } from '../../tools/debugTools';
+import { clearDatabase, createTestUsers, CreateTestUsersOptions } from '../../tools/debugTools';
 import { bodyFields } from '../../utils/requestUtils';
 import Router from '../../utils/Router';
 import { RouteType } from '../../utils/types';
@@ -12,6 +12,8 @@ router.public = true;
 
 interface Query {
 	action: string;
+	count?: number;
+	fromNum?: number;
 }
 
 router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
@@ -20,7 +22,16 @@ router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
 	console.info(`Action: ${query.action}`);
 
 	if (query.action === 'createTestUsers') {
-		await createTestUsers(ctx.joplin.db, config());
+		const options: CreateTestUsersOptions = {};
+
+		if ('count' in query) options.count = query.count;
+		if ('fromNum' in query) options.fromNum = query.fromNum;
+
+		await createTestUsers(ctx.joplin.db, config(), options);
+	}
+
+	if (query.action === 'clearDatabase') {
+		await clearDatabase(ctx.joplin.db);
 	}
 });
 

packages/server/src/routes/index/tasks.ts

@@ -0,0 +1,135 @@
+import { makeUrl, redirect, SubPath, UrlType } from '../../utils/routeUtils';
+import Router from '../../utils/Router';
+import { RouteType } from '../../utils/types';
+import { AppContext } from '../../utils/types';
+import { bodyFields } from '../../utils/requestUtils';
+import { ErrorBadRequest, ErrorForbidden } from '../../utils/errors';
+import defaultView from '../../utils/defaultView';
+import { makeTableView, Row, Table } from '../../utils/views/table';
+import { yesOrNo } from '../../utils/strings';
+import { formatDateTime } from '../../utils/time';
+import { createCsrfTag } from '../../utils/csrf';
+import { RunType } from '../../services/TaskService';
+import { NotificationKey } from '../../models/NotificationModel';
+import { NotificationLevel } from '../../services/database/types';
+
+const router: Router = new Router(RouteType.Web);
+
+router.post('tasks', async (_path: SubPath, ctx: AppContext) => {
+	const user = ctx.joplin.owner;
+	if (!user.is_admin) throw new ErrorForbidden();
+
+	const taskService = ctx.joplin.services.tasks;
+	const fields: any = await bodyFields(ctx.req);
+
+	if (fields.startTaskButton) {
+		const errors: Error[] = [];
+
+		for (const k of Object.keys(fields)) {
+			if (k.startsWith('checkbox_')) {
+				const taskId = k.substr(9);
+				try {
+					void taskService.runTask(taskId, RunType.Manual);
+				} catch (error) {
+					errors.push(error);
+				}
+			}
+		}
+
+		if (errors.length) {
+			await ctx.joplin.models.notification().add(
+				user.id,
+				NotificationKey.Any,
+				NotificationLevel.Error,
+				`Some tasks could not be started: ${errors.join('. ')}`
+			);
+		}
+	} else {
+		throw new ErrorBadRequest('Invalid action');
+	}
+
+	return redirect(ctx, makeUrl(UrlType.Tasks));
+});
+
+router.get('tasks', async (_path: SubPath, ctx: AppContext) => {
+	const user = ctx.joplin.owner;
+	if (!user.is_admin) throw new ErrorForbidden();
+
+	const taskService = ctx.joplin.services.tasks;
+
+	const taskRows: Row[] = [];
+	for (const [taskId, task] of Object.entries(taskService.tasks)) {
+		const state = taskService.taskState(taskId);
+
+		taskRows.push([
+			{
+				value: `checkbox_${taskId}`,
+				checkbox: true,
+			},
+			{
+				value: taskId,
+			},
+			{
+				value: task.description,
+			},
+			{
+				value: task.schedule,
+			},
+			{
+				value: yesOrNo(state.running),
+			},
+			{
+				value: state.lastRunTime ? formatDateTime(state.lastRunTime) : '-',
+			},
+			{
+				value: state.lastCompletionTime ? formatDateTime(state.lastCompletionTime) : '-',
+			},
+		]);
+	}
+
+	const table: Table = {
+		headers: [
+			{
+				name: 'select',
+				label: '',
+			},
+			{
+				name: 'id',
+				label: 'ID',
+			},
+			{
+				name: 'description',
+				label: 'Description',
+			},
+			{
+				name: 'schedule',
+				label: 'Schedule',
+			},
+			{
+				name: 'running',
+				label: 'Running',
+			},
+			{
+				name: 'lastRunTime',
+				label: 'Last Run',
+			},
+			{
+				name: 'lastCompletionTime',
+				label: 'Last Completion',
+			},
+		],
+		rows: taskRows,
+	};
+
+	return {
+		...defaultView('tasks', 'Tasks'),
+		content: {
+			itemTable: makeTableView(table),
+			postUrl: makeUrl(UrlType.Tasks),
+			csrfTag: await createCsrfTag(ctx),
+		},
+		cssFiles: ['index/tasks'],
+	};
+});
+
+export default router;

packages/server/src/routes/routes.ts

@@ -1,31 +1,32 @@
 import { Routers } from '../utils/routeUtils';
 
 import apiBatch from './api/batch';
+import apiBatchItems from './api/batch_items';
 import apiDebug from './api/debug';
 import apiEvents from './api/events';
-import apiBatchItems from './api/batch_items';
 import apiItems from './api/items';
 import apiPing from './api/ping';
 import apiSessions from './api/sessions';
-import apiUsers from './api/users';
 import apiShares from './api/shares';
 import apiShareUsers from './api/share_users';
+import apiUsers from './api/users';
 
 import indexChanges from './index/changes';
+import indexHelp from './index/help';
 import indexHome from './index/home';
 import indexItems from './index/items';
 import indexLogin from './index/login';
 import indexLogout from './index/logout';
 import indexNotifications from './index/notifications';
 import indexPassword from './index/password';
-import indexSignup from './index/signup';
-import indexShares from './index/shares';
-import indexUsers from './index/users';
-import indexStripe from './index/stripe';
-import indexTerms from './index/terms';
 import indexPrivacy from './index/privacy';
+import indexShares from './index/shares';
+import indexSignup from './index/signup';
+import indexStripe from './index/stripe';
+import indexTasks from './index/tasks';
+import indexTerms from './index/terms';
 import indexUpgrade from './index/upgrade';
-import indexHelp from './index/help';
+import indexUsers from './index/users';
 
 import defaultRoute from './default';
 
@@ -56,6 +57,7 @@ const routes: Routers = {
 	'privacy': indexPrivacy,
 	'upgrade': indexUpgrade,
 	'help': indexHelp,
+	'tasks': indexTasks,
 
 	'': defaultRoute,
 };

packages/server/src/services/CronService.ts

@@ -1,42 +0,0 @@
-import Logger from '@joplin/lib/Logger';
-import BaseService from './BaseService';
-const cron = require('node-cron');
-
-const logger = Logger.create('cron');
-
-async function runCronTask(name: string, callback: Function) {
-	const startTime = Date.now();
-	logger.info(`Running task "${name}"`);
-	try {
-		await callback();
-	} catch (error) {
-		logger.error(`On task "${name}"`, error);
-	}
-	logger.info(`Completed task "${name}" in ${Date.now() - startTime}ms`);
-}
-
-export default class CronService extends BaseService {
-
-	public async runInBackground() {
-		cron.schedule('0 */6 * * *', async () => {
-			await runCronTask('deleteExpiredTokens', async () => this.models.token().deleteExpiredTokens());
-		});
-
-		cron.schedule('0 * * * *', async () => {
-			await runCronTask('updateTotalSizes', async () => this.models.item().updateTotalSizes());
-		});
-
-		cron.schedule('0 12 * * *', async () => {
-			await runCronTask('handleBetaUserEmails', async () => this.models.user().handleBetaUserEmails());
-		});
-
-		cron.schedule('0 13 * * *', async () => {
-			await runCronTask('handleFailedPaymentSubscriptions', async () => this.models.user().handleFailedPaymentSubscriptions());
-		});
-
-		cron.schedule('0 14 * * *', async () => {
-			await runCronTask('handleOversizedAccounts', async () => this.models.user().handleOversizedAccounts());
-		});
-	}
-
-}

packages/server/src/services/TaskService.test.ts

@@ -0,0 +1,82 @@
+import config from '../config';
+import { Models } from '../models/factory';
+import { afterAllTests, beforeAllDb, beforeEachDb, expectThrow, models, msleep } from '../utils/testing/testUtils';
+import { Env } from '../utils/types';
+import TaskService, { RunType, Task } from './TaskService';
+
+const newService = () => {
+	return new TaskService(Env.Dev, models(), config());
+};
+
+describe('TaskService', function() {
+
+	beforeAll(async () => {
+		await beforeAllDb('TaskService');
+	});
+
+	afterAll(async () => {
+		await afterAllTests();
+	});
+
+	beforeEach(async () => {
+		await beforeEachDb();
+	});
+
+	test('should register a task', async function() {
+		const service = newService();
+
+		const task: Task = {
+			id: 'test',
+			description: '',
+			run: (_models: Models) => {},
+			schedule: '',
+		};
+
+		service.registerTask(task);
+
+		expect(service.tasks['test']).toBeTruthy();
+		await expectThrow(async () => service.registerTask(task));
+	});
+
+	test('should run a task', async function() {
+		const service = newService();
+
+		let finishTask = false;
+		let taskHasRan = false;
+
+		const task: Task = {
+			id: 'test',
+			description: '',
+			run: async (_models: Models) => {
+				const iid = setInterval(() => {
+					if (finishTask) {
+						clearInterval(iid);
+						taskHasRan = true;
+					}
+				}, 1);
+			},
+			schedule: '',
+		};
+
+		service.registerTask(task);
+
+		expect(service.taskState('test').running).toBe(false);
+
+		const startTime = new Date();
+
+		void service.runTask('test', RunType.Manual);
+		expect(service.taskState('test').running).toBe(true);
+		expect(service.taskState('test').lastCompletionTime).toBeFalsy();
+		expect(service.taskState('test').lastRunTime.getTime()).toBeGreaterThanOrEqual(startTime.getTime());
+
+		await msleep(1);
+		finishTask = true;
+		await msleep(3);
+
+		expect(taskHasRan).toBe(true);
+		expect(service.taskState('test').running).toBe(false);
+		expect(service.taskState('test').lastCompletionTime.getTime()).toBeGreaterThan(startTime.getTime());
+
+	});
+
+});

packages/server/src/services/TaskService.ts

@@ -0,0 +1,108 @@
+import Logger from '@joplin/lib/Logger';
+import { Models } from '../models/factory';
+import BaseService from './BaseService';
+const cron = require('node-cron');
+
+const logger = Logger.create('TaskService');
+
+type TaskId = string;
+
+export enum RunType {
+	Scheduled = 1,
+	Manual = 2,
+}
+
+const runTypeToString = (runType: RunType) => {
+	if (runType === RunType.Scheduled) return 'scheduled';
+	if (runType === RunType.Manual) return 'manual';
+	throw new Error(`Unknown run type: ${runType}`);
+};
+
+export interface Task {
+	id: TaskId;
+	description: string;
+	schedule: string;
+	run(models: Models): void;
+}
+
+export type Tasks = Record<TaskId, Task>;
+
+interface TaskState {
+	running: boolean;
+	lastRunTime: Date;
+	lastCompletionTime: Date;
+}
+
+const defaultTaskState: TaskState = {
+	running: false,
+	lastRunTime: null,
+	lastCompletionTime: null,
+};
+
+export default class TaskService extends BaseService {
+
+	private tasks_: Tasks = {};
+	private taskStates_: Record<TaskId, TaskState> = {};
+
+	public registerTask(task: Task) {
+		if (this.tasks_[task.id]) throw new Error(`Already a task with this ID: ${task.id}`);
+		this.tasks_[task.id] = task;
+		this.taskStates_[task.id] = { ...defaultTaskState };
+	}
+
+	public registerTasks(tasks: Task[]) {
+		for (const task of tasks) this.registerTask(task);
+	}
+
+	public get tasks(): Tasks {
+		return this.tasks_;
+	}
+
+	public taskState(id: TaskId): TaskState {
+		if (!this.taskStates_[id]) throw new Error(`No such task: ${id}`);
+		return this.taskStates_[id];
+	}
+
+	// TODO: add tests
+
+	public async runTask(id: TaskId, runType: RunType) {
+		const state = this.taskState(id);
+		if (state.running) throw new Error(`Task is already running: ${id}`);
+
+		const startTime = Date.now();
+
+		this.taskStates_[id] = {
+			...this.taskStates_[id],
+			running: true,
+			lastRunTime: new Date(),
+		};
+
+		try {
+			logger.info(`Running "${id}" (${runTypeToString(runType)})...`);
+			await this.tasks_[id].run(this.models);
+		} catch (error) {
+			logger.error(`On task "${id}"`, error);
+		}
+
+		this.taskStates_[id] = {
+			...this.taskStates_[id],
+			running: false,
+			lastCompletionTime: new Date(),
+		};
+
+		logger.info(`Completed "${id}" in ${Date.now() - startTime}ms`);
+	}
+
+	public async runInBackground() {
+		for (const [taskId, task] of Object.entries(this.tasks_)) {
+			if (!task.schedule) continue;
+
+			logger.info(`Scheduling task "${taskId}": ${task.schedule}`);
+
+			cron.schedule(task.schedule, async () => {
+				await this.runTask(taskId, RunType.Scheduled);
+			});
+		}
+	}
+
+}

packages/server/src/services/database/types.ts

@@ -6,6 +6,7 @@ export enum ItemAddressingType {
 }
 
 export enum NotificationLevel {
+	Error = 5,
 	Important = 10,
 	Normal = 20,
 }

packages/server/src/services/types.ts

@@ -1,11 +1,11 @@
-import CronService from './CronService';
 import EmailService from './EmailService';
 import MustacheService from './MustacheService';
 import ShareService from './ShareService';
+import TaskService from './TaskService';
 
 export interface Services {
 	share: ShareService;
 	email: EmailService;
-	cron: CronService;
 	mustache: MustacheService;
+	tasks: TaskService;
 }

packages/server/src/tools/debugTools.ts

@@ -1,9 +1,14 @@
 import { DbConnection, dropTables, migrateLatest } from '../db';
 import newModelFactory from '../models/factory';
 import { AccountType } from '../models/UserModel';
-import { UserFlagType } from '../services/database/types';
+import { User, UserFlagType } from '../services/database/types';
 import { Config } from '../utils/types';
 
+export interface CreateTestUsersOptions {
+	count?: number;
+	fromNum?: number;
+}
+
 export async function handleDebugCommands(argv: any, db: DbConnection, config: Config): Promise<boolean> {
 	if (argv.debugCreateTestUsers) {
 		await createTestUsers(db, config);
@@ -14,51 +19,79 @@ export async function handleDebugCommands(argv: any, db: DbConnection, config: C
 	return true;
 }
 
-export async function createTestUsers(db: DbConnection, config: Config) {
+export async function clearDatabase(db: DbConnection) {
 	await dropTables(db);
 	await migrateLatest(db);
+}
+
+export async function createTestUsers(db: DbConnection, config: Config, options: CreateTestUsersOptions = null) {
+	options = {
+		count: 0,
+		fromNum: 1,
+		...options,
+	};
 
 	const password = 'hunter1hunter2hunter3';
-	const models = newModelFactory(db, config);
 
-	for (let userNum = 1; userNum <= 2; userNum++) {
-		await models.user().save({
-			email: `user${userNum}@example.com`,
-			password,
-			full_name: `User ${userNum}`,
-		});
-	}
+	if (options.count) {
+		const models = newModelFactory(db, config);
 
-	{
-		const { user } = await models.subscription().saveUserAndSubscription(
-			'usersub@example.com',
-			'With Sub',
-			AccountType.Basic,
-			'usr_111',
-			'sub_111'
-		);
-		await models.user().save({ id: user.id, password });
-	}
+		const users: User[] = [];
 
-	{
-		const { user, subscription } = await models.subscription().saveUserAndSubscription(
-			'userfailedpayment@example.com',
-			'Failed Payment',
-			AccountType.Basic,
-			'usr_222',
-			'sub_222'
-		);
-		await models.user().save({ id: user.id, password });
-		await models.subscription().handlePayment(subscription.stripe_subscription_id, false);
-	}
+		for (let i = 0; i < options.count; i++) {
+			const userNum = i + options.fromNum;
+			users.push({
+				email: `user${userNum}@example.com`,
+				password,
+				full_name: `User ${userNum}`,
+			});
+		}
 
-	{
-		const user = await models.user().save({
-			email: 'userwithflags@example.com',
-			password,
-			full_name: 'User Withflags',
-		});
+		await models.user().saveMulti(users);
+	} else {
+		await dropTables(db);
+		await migrateLatest(db);
+		const models = newModelFactory(db, config);
 
-		await models.userFlag().add(user.id, UserFlagType.AccountOverLimit);
+		for (let userNum = 1; userNum <= 2; userNum++) {
+			await models.user().save({
+				email: `user${userNum}@example.com`,
+				password,
+				full_name: `User ${userNum}`,
+			});
+		}
+
+		{
+			const { user } = await models.subscription().saveUserAndSubscription(
+				'usersub@example.com',
+				'With Sub',
+				AccountType.Basic,
+				'usr_111',
+				'sub_111'
+			);
+			await models.user().save({ id: user.id, password });
+		}
+
+		{
+			const { user, subscription } = await models.subscription().saveUserAndSubscription(
+				'userfailedpayment@example.com',
+				'Failed Payment',
+				AccountType.Basic,
+				'usr_222',
+				'sub_222'
+			);
+			await models.user().save({ id: user.id, password });
+			await models.subscription().handlePayment(subscription.stripe_subscription_id, false);
+		}
+
+		{
+			const user = await models.user().save({
+				email: 'userwithflags@example.com',
+				password,
+				full_name: 'User Withflags',
+			});
+
+			await models.userFlag().add(user.id, UserFlagType.AccountOverLimit);
+		}
 	}
 }

packages/server/src/utils/routeUtils.ts

@@ -271,6 +271,7 @@ export enum UrlType {
 	Login = 'login',
 	Terms = 'terms',
 	Privacy = 'privacy',
+	Tasks = 'tasks',
 }
 
 export function makeUrl(urlType: UrlType): string {

packages/server/src/utils/setupAppContext.ts

@@ -7,15 +7,15 @@ import routes from '../routes/routes';
 import ShareService from '../services/ShareService';
 import { Services } from '../services/types';
 import EmailService from '../services/EmailService';
-import CronService from '../services/CronService';
 import MustacheService from '../services/MustacheService';
+import setupTaskService from './setupTaskService';
 
 async function setupServices(env: Env, models: Models, config: Config): Promise<Services> {
 	const output: Services = {
 		share: new ShareService(env, models, config),
 		email: new EmailService(env, models, config),
-		cron: new CronService(env, models, config),
 		mustache: new MustacheService(config.viewDir, config.baseUrl),
+		tasks: setupTaskService(env, models, config),
 	};
 
 	await output.mustache.loadPartials();

packages/server/src/utils/setupTaskService.ts

@@ -0,0 +1,49 @@
+import { Models } from '../models/factory';
+import TaskService, { Task } from '../services/TaskService';
+import { Config, Env } from './types';
+
+export default function(env: Env, models: Models, config: Config): TaskService {
+	const taskService = new TaskService(env, models, config);
+
+	let tasks: Task[] = [
+		{
+			id: 'deleteExpiredTokens',
+			description: 'Delete expired tokens',
+			schedule: '0 */6 * * *',
+			run: (models: Models) => models.token().deleteExpiredTokens(),
+		},
+		{
+			id: 'updateTotalSizes',
+			description: 'Update total sizes',
+			schedule: '0 * * * *',
+			run: (models: Models) => models.item().updateTotalSizes(),
+		},
+		{
+			id: 'handleOversizedAccounts',
+			description: 'Process oversized accounts',
+			schedule: '0 14 * * *',
+			run: (models: Models) => models.user().handleOversizedAccounts(),
+		},
+	];
+
+	if (config.isJoplinCloud) {
+		tasks = tasks.concat([
+			{
+				id: 'handleBetaUserEmails',
+				description: 'Process beta user emails',
+				schedule: '0 12 * * *',
+				run: (models: Models) => models.user().handleBetaUserEmails(),
+			},
+			{
+				id: 'handleFailedPaymentSubscriptions',
+				description: 'Process failed payment subscriptions',
+				schedule: '0 13 * * *',
+				run: (models: Models) => models.user().handleFailedPaymentSubscriptions(),
+			},
+		]);
+	}
+
+	taskService.registerTasks(tasks);
+
+	return taskService;
+}

packages/server/src/utils/startServices.ts

@@ -3,5 +3,5 @@ import { Services } from '../services/types';
 export default async function startServices(services: Services) {
 	void services.share.runInBackground();
 	void services.email.runInBackground();
-	void services.cron.runInBackground();
+	void services.tasks.runInBackground();
 }

packages/server/src/utils/time.ts

@@ -29,7 +29,8 @@ export function msleep(ms: number) {
 	});
 }
 
-export function formatDateTime(ms: number): string {
+export function formatDateTime(ms: number | Date): string {
+	ms = ms instanceof Date ? ms.getTime() : ms;
 	return `${dayjs(ms).format('D MMM YY HH:mm:ss')} (${defaultTimezone()})`;
 }
 

packages/server/src/utils/types.ts

@@ -17,7 +17,7 @@ export enum Env {
 export interface NotificationView {
 	id: Uuid;
 	messageHtml: string;
-	level: string;
+	levelClassName: string;
 	closeUrl: string;
 }
 

packages/server/src/utils/views/table.ts

@@ -4,11 +4,13 @@ import { setQueryParameters } from '../urlUtils';
 const defaultSortOrder = PaginationOrderDir.ASC;
 
 function headerIsSelectedClass(name: string, pagination: Pagination): string {
+	if (!pagination) return '';
 	const orderBy = pagination.order[0].by;
 	return name === orderBy ? 'is-selected' : '';
 }
 
 function headerSortIconDir(name: string, pagination: Pagination): string {
+	if (!pagination) return '';
 	const orderBy = pagination.order[0].by;
 	const orderDir = orderBy === name ? pagination.order[0].dir : defaultSortOrder;
 	return orderDir === PaginationOrderDir.ASC ? 'up' : 'down';
@@ -35,6 +37,7 @@ interface HeaderView {
 
 interface RowItem {
 	value: string;
+	checkbox?: boolean;
 	url?: string;
 	stretch?: boolean;
 }
@@ -45,6 +48,7 @@ interface RowItemView {
 	value: string;
 	classNames: string[];
 	url: string;
+	checkbox: boolean;
 }
 
 type RowView = RowItemView[];
@@ -52,10 +56,10 @@ type RowView = RowItemView[];
 export interface Table {
 	headers: Header[];
 	rows: Row[];
-	baseUrl: string;
-	requestQuery: any;
-	pageCount: number;
-	pagination: Pagination;
+	baseUrl?: string;
+	requestQuery?: any;
+	pageCount?: number;
+	pagination?: Pagination;
 }
 
 export interface TableView {
@@ -77,7 +81,7 @@ export function makeTablePagination(query: any, defaultOrderField: string, defau
 function makeHeaderView(header: Header, parentBaseUrl: string, baseUrlQuery: PaginationQueryParams, pagination: Pagination): HeaderView {
 	return {
 		label: header.label,
-		sortLink: setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }),
+		sortLink: !pagination ? null : setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }),
 		classNames: [header.stretch ? 'stretch' : 'nowrap', headerIsSelectedClass(header.name, pagination)],
 		iconDir: headerSortIconDir(header.name, pagination),
 	};
@@ -89,14 +93,21 @@ function makeRowView(row: Row): RowView {
 			value: rowItem.value,
 			classNames: [rowItem.stretch ? 'stretch' : 'nowrap'],
 			url: rowItem.url,
+			checkbox: rowItem.checkbox,
 		};
 	});
 }
 
 export function makeTableView(table: Table): TableView {
-	const baseUrlQuery = filterPaginationQueryParams(table.requestQuery);
-	const pagination = table.pagination;
-	const paginationLinks = createPaginationLinks(pagination.page, table.pageCount, setQueryParameters(table.baseUrl, { ...baseUrlQuery, 'page': 'PAGE_NUMBER' }));
+	let paginationLinks: PageLink[] = [];
+	let baseUrlQuery: PaginationQueryParams = null;
+	let pagination: Pagination = null;
+
+	if (table.pageCount) {
+		baseUrlQuery = filterPaginationQueryParams(table.requestQuery);
+		pagination = table.pagination;
+		paginationLinks = createPaginationLinks(pagination.page, table.pageCount, setQueryParameters(table.baseUrl, { ...baseUrlQuery, 'page': 'PAGE_NUMBER' }));
+	}
 
 	return {
 		headers: table.headers.map(h => makeHeaderView(h, table.baseUrl, baseUrlQuery, pagination)),

packages/server/src/views/index/tasks.mustache

@@ -0,0 +1,11 @@
+<form method='POST' action="{{postUrl}}">
+	{{{csrfTag}}}
+
+	{{#itemTable}}
+		{{>table}}
+	{{/itemTable}}
+
+	<div class="block">
+		<input class="button is-link" type="submit" value="Start selected tasks" name="startTaskButton"/>
+	</div>
+</form>

packages/server/src/views/partials/navbar.mustache

@@ -16,6 +16,9 @@
 						{{/global.owner.is_admin}}
 						<a class="navbar-item" href="{{{global.baseUrl}}}/items">Items</a>
 						<a class="navbar-item" href="{{{global.baseUrl}}}/changes">Log</a>
+						{{#global.owner.is_admin}}
+							<a class="navbar-item" href="{{{global.baseUrl}}}/tasks">Tasks</a>
+						{{/global.owner.is_admin}}
 					</div>
 					<div class="navbar-end">
 						{{#global.isJoplinCloud}}

packages/server/src/views/partials/notifications.mustache

@@ -1,6 +1,6 @@
 {{#global.hasNotifications}}
 	{{#global.notifications}}
-	<div class="notification is-{{level}}" id="notification-{{id}}">
+	<div class="notification {{levelClassName}}" id="notification-{{id}}">
 		<button data-close-url="{{closeUrl}}" data-id="{{id}}" class="delete close-notification-button"></button>
 		{{{messageHtml}}}
 	</div>

packages/server/src/views/partials/table.mustache

@@ -1,20 +1,22 @@
-<table class="table is-fullwidth is-hoverable">
-	<thead>
-		<tr>
-			{{#headers}}
-				{{>tableHeader}}
-			{{/headers}}
-		</tr>
-	</thead>
-	<tbody>
-		{{#rows}}
+<div class="table-container">
+	<table class="table is-fullwidth is-hoverable ">
+		<thead>
 			<tr>
-				{{#.}}
-					{{>tableRowItem}}
-				{{/.}}
+				{{#headers}}
+					{{>tableHeader}}
+				{{/headers}}
 			</tr>
-		{{/rows}}
-	</tbody>
-</table>
+		</thead>
+		<tbody>
+			{{#rows}}
+				<tr>
+					{{#.}}
+						{{>tableRowItem}}
+					{{/.}}
+				</tr>
+			{{/rows}}
+		</tbody>
+	</table>
+</div>
 
 {{>pagination}}

packages/server/src/views/partials/tableHeader.mustache

@@ -1,3 +1,8 @@
 <th class="{{#classNames}}{{.}} {{/classNames}}">
-	<a href="{{sortLink}}" class="sort-button">{{label}} <i class="fas fa-caret-{{iconDir}}"></i></a>
+	{{#sortLink}}
+		<a href="{{sortLink}}" class="sort-button">{{label}} <i class="fas fa-caret-{{iconDir}}"></i></a>
+	{{/sortLink}}
+	{{^sortLink}}
+		{{label}}
+	{{/sortLink}}
 </th>

packages/server/src/views/partials/tableRowItem.mustache

@@ -1,3 +1,8 @@
 <td class="{{#classNames}}{{.}} {{/classNames}}">
-	{{#url}}<a href="{{.}}"></span>{{/url}}{{value}}</a>
+	{{#checkbox}}
+		<input type="checkbox" name="{{value}}"/>
+	{{/checkbox}}
+	{{^checkbox}}
+		{{#url}}<a href="{{.}}"></span>{{/url}}{{value}}</a>
+	{{/checkbox}}
 </td>