Tools: Added tools to stress test Joplin Server

.eslintignore

@@ -76,6 +76,9 @@ packages/app-cli/app/command-e2ee.js.map
 packages/app-cli/app/command-settingschema.d.ts
 packages/app-cli/app/command-settingschema.js
 packages/app-cli/app/command-settingschema.js.map
+packages/app-cli/app/command-testing.d.ts
+packages/app-cli/app/command-testing.js
+packages/app-cli/app/command-testing.js.map
 packages/app-cli/app/services/plugins/PluginRunner.d.ts
 packages/app-cli/app/services/plugins/PluginRunner.js
 packages/app-cli/app/services/plugins/PluginRunner.js.map
@@ -109,6 +112,9 @@ packages/app-cli/tests/services/plugins/sandboxProxy.js.map
 packages/app-cli/tests/testUtils.d.ts
 packages/app-cli/tests/testUtils.js
 packages/app-cli/tests/testUtils.js.map
+packages/app-cli/tools/populateDatabase.d.ts
+packages/app-cli/tools/populateDatabase.js
+packages/app-cli/tools/populateDatabase.js.map
 packages/app-desktop/ElectronAppWrapper.d.ts
 packages/app-desktop/ElectronAppWrapper.js
 packages/app-desktop/ElectronAppWrapper.js.map

.gitignore

@@ -61,6 +61,9 @@ packages/app-cli/app/command-e2ee.js.map
 packages/app-cli/app/command-settingschema.d.ts
 packages/app-cli/app/command-settingschema.js
 packages/app-cli/app/command-settingschema.js.map
+packages/app-cli/app/command-testing.d.ts
+packages/app-cli/app/command-testing.js
+packages/app-cli/app/command-testing.js.map
 packages/app-cli/app/services/plugins/PluginRunner.d.ts
 packages/app-cli/app/services/plugins/PluginRunner.js
 packages/app-cli/app/services/plugins/PluginRunner.js.map
@@ -94,6 +97,9 @@ packages/app-cli/tests/services/plugins/sandboxProxy.js.map
 packages/app-cli/tests/testUtils.d.ts
 packages/app-cli/tests/testUtils.js
 packages/app-cli/tests/testUtils.js.map
+packages/app-cli/tools/populateDatabase.d.ts
+packages/app-cli/tools/populateDatabase.js
+packages/app-cli/tools/populateDatabase.js.map
 packages/app-desktop/ElectronAppWrapper.d.ts
 packages/app-desktop/ElectronAppWrapper.js
 packages/app-desktop/ElectronAppWrapper.js.map

packages/app-cli/.gitignore

@@ -24,3 +24,5 @@ tests/support/nextcloud-auth.json
 tests/support/onedrive-auth.txt
 build/
 patches/
+createUsers-*.txt
+tools/temp/

packages/app-cli/app/cli-utils.js

@@ -89,7 +89,7 @@ cliUtils.makeCommandArgs = function(cmd, argv) {
 		flags = cliUtils.parseFlags(flags);
 
 		if (!flags.arg) {
-			booleanFlags.push(flags.short);
+			if (flags.short) booleanFlags.push(flags.short);
 			if (flags.long) booleanFlags.push(flags.long);
 		}
 

packages/app-cli/app/command-testing.ts

@@ -0,0 +1,95 @@
+const { BaseCommand } = require('./base-command.js');
+import { reg } from '@joplin/lib/registry';
+import Note from '@joplin/lib/models/Note';
+import uuid from '@joplin/lib/uuid';
+import populateDatabase from '@joplin/lib/services/debug/populateDatabase';
+
+function randomElement(array: any[]): any {
+	if (!array.length) return null;
+	return array[Math.floor(Math.random() * array.length)];
+}
+
+function itemCount(args: any) {
+	const count = Number(args.arg0);
+	if (!count || isNaN(count)) throw new Error('Note count must be specified');
+	return count;
+}
+
+class Command extends BaseCommand {
+	usage() {
+		return 'testing <command> [arg0]';
+	}
+
+	description() {
+		return 'testing';
+	}
+
+	enabled() {
+		return false;
+	}
+
+	options(): any[] {
+		return [
+			['--folder-count <count>', 'Folders to create'],
+			['--note-count <count>', 'Notes to create'],
+			['--tag-count <count>', 'Tags to create'],
+			['--tags-per-note <count>', 'Tags per note'],
+			['--silent', 'Silent'],
+		];
+	}
+
+	async action(args: any) {
+		const { command, options } = args;
+
+		if (command === 'populate') {
+			await populateDatabase(reg.db(), {
+				folderCount: options['folder-count'],
+				noteCount: options['note-count'],
+				tagCount: options['tag-count'],
+				tagsPerNote: options['tags-per-note'],
+				silent: options['silent'],
+			});
+		}
+
+		const promises: any[] = [];
+
+		if (command === 'createRandomNotes') {
+			const noteCount = itemCount(args);
+
+			for (let i = 0; i < noteCount; i++) {
+				promises.push(Note.save({
+					title: `Note ${uuid.createNano()}`,
+				}));
+			}
+		}
+
+		if (command === 'updateRandomNotes') {
+			const noteCount = itemCount(args);
+
+			const noteIds = await Note.allIds();
+
+			for (let i = 0; i < noteCount; i++) {
+				const noteId = randomElement(noteIds);
+				promises.push(Note.save({
+					id: noteId,
+					title: `Note ${uuid.createNano()}`,
+				}));
+			}
+		}
+
+		if (command === 'deleteRandomNotes') {
+			const noteCount = itemCount(args);
+			const noteIds = await Note.allIds();
+
+			for (let i = 0; i < noteCount; i++) {
+				const noteId = randomElement(noteIds);
+				promises.push(Note.delete(noteId));
+			}
+		}
+
+		await Promise.all(promises);
+	}
+
+}
+
+module.exports = Command;

packages/app-cli/createUsers.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Start the server with:
+#
+# JOPLIN_IS_TESTING=1 npm run start-dev
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+# curl --data '{"action": "clearDatabase"}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
+
+# SMALL
+
+# curl --data '{"action": "createTestUsers", "count": 400, "fromNum": 1}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
+
+NUM=398
+while [ "$NUM" -lt 400 ]; do
+	NUM=$(( NUM + 1 ))
+
+	echo "User $NUM"
+
+	CMD_FILE="$SCRIPT_DIR/createUsers-$NUM.txt"
+	PROFILE_DIR=~/.config/joplindev-testing-$NUM
+	USER_EMAIL="user$NUM@example.com"
+
+	rm -rf "$CMD_FILE" "$PROFILE_DIR"
+	touch "$CMD_FILE"
+
+	FLAG_FOLDER_COUNT=100
+	FLAG_NOTE_COUNT=1000
+	FLAG_TAG_COUNT=20
+
+	if [ "$NUM" -gt 300 ]; then
+		FLAG_FOLDER_COUNT=2000
+		FLAG_NOTE_COUNT=10000
+		FLAG_TAG_COUNT=200
+	fi
+
+	if [ "$NUM" -gt 399 ]; then
+		FLAG_FOLDER_COUNT=10000
+		FLAG_NOTE_COUNT=150000
+		FLAG_TAG_COUNT=2000
+	fi
+
+	echo "testing populate --silent --folder-count $FLAG_FOLDER_COUNT --note-count $FLAG_NOTE_COUNT --tag-count $FLAG_TAG_COUNT" >> "$CMD_FILE"
+	echo "config keychain.supported 0" >> "$CMD_FILE" 
+	echo "config sync.target 10" >> "$CMD_FILE" 
+	echo "config sync.10.username $USER_EMAIL" >> "$CMD_FILE" 
+	echo "config sync.10.password hunter1hunter2hunter3" >> "$CMD_FILE"
+	echo "sync" >> "$CMD_FILE"
+
+	npm start -- --profile "$PROFILE_DIR" batch "$CMD_FILE"
+done

packages/app-cli/package.json

@@ -10,6 +10,7 @@
     "test-ci": "jest --config=jest.config.js --forceExit",
     "build": "gulp build",
     "start": "gulp build -L && node \"build/main.js\" --stack-trace-enabled --log-level debug --env dev",
+    "start-no-build": "node \"build/main.js\" --stack-trace-enabled --log-level debug --env dev",
     "tsc": "node node_modules/typescript/bin/tsc --project tsconfig.json",
     "watch": "node node_modules/typescript/bin/tsc --watch --project tsconfig.json"
   },

packages/app-cli/tools/populateDatabase.ts

@@ -0,0 +1,86 @@
+import * as fs from 'fs-extra';
+import { homedir } from 'os';
+import { execCommand2 } from '@joplin/tools/tool-utils';
+import { chdir } from 'process';
+
+const minUserNum = 1;
+const maxUserNum = 400;
+
+const cliDir = `${__dirname}/..`;
+const tempDir = `${__dirname}/temp`;
+
+function randomInt(min: number, max: number) {
+	min = Math.ceil(min);
+	max = Math.floor(max);
+	return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+const processing_: Record<number, boolean> = {};
+
+const processUser = async (userNum: number) => {
+	if (processing_[userNum]) {
+		console.info(`User already being processed: ${userNum} - skipping`);
+		return;
+	}
+
+	processing_[userNum] = true;
+
+	try {
+		const userEmail = `user${userNum}@example.com`;
+		const userPassword = 'hunter1hunter2hunter3';
+		const commandFile = `${tempDir}/populateDatabase-${userNum}.txt`;
+		const profileDir = `${homedir()}/.config/joplindev-populate/joplindev-testing-${userNum}`;
+
+		const commands: string[] = [];
+		const jackpot = Math.random() >= 0.95 ? 100 : 1;
+
+		commands.push(`testing createRandomNotes ${randomInt(1, 500 * jackpot)}`);
+		commands.push(`testing updateRandomNotes ${randomInt(1, 1500 * jackpot)}`);
+		commands.push(`testing deleteRandomNotes ${randomInt(1, 200 * jackpot)}`);
+		commands.push('config keychain.supported 0');
+		commands.push('config sync.target 10');
+		commands.push(`config sync.10.username ${userEmail}`);
+		commands.push(`config sync.10.password ${userPassword}`);
+		commands.push('sync');
+
+		await fs.writeFile(commandFile, commands.join('\n'), 'utf8');
+
+		await chdir(cliDir);
+
+		await execCommand2(['npm', 'run', 'start-no-build', '--', '--profile', profileDir, 'batch', commandFile]);
+	} catch (error) {
+		console.error(`Could not process user ${userNum}:`, error);
+	} finally {
+		delete processing_[userNum];
+	}
+};
+
+const waitForProcessing = (count: number) => {
+	return new Promise((resolve) => {
+		const iid = setInterval(() => {
+			if (Object.keys(processing_).length <= count) {
+				clearInterval(iid);
+				resolve(null);
+			}
+		}, 100);
+	});
+};
+
+const main = async () => {
+	await fs.mkdirp(tempDir);
+
+	// Build the app once before starting, because we'll use start-no-build to
+	// run the scripts (faster)
+	await execCommand2(['npm', 'run', 'build']);
+
+	while (true) {
+		const userNum = randomInt(minUserNum, maxUserNum);
+		void processUser(userNum);
+		await waitForProcessing(10);
+	}
+};
+
+main().catch((error) => {
+	console.error('Fatal error', error);
+	process.exit(1);
+});

packages/server/src/models/UserModel.ts

@@ -469,4 +469,12 @@ export default class UserModel extends BaseModel<User> {
 		}, 'UserModel::save');
 	}
 
+	public async saveMulti(users: User[], options: SaveOptions = {}): Promise<void> {
+		await this.withTransaction(async () => {
+			for (const user of users) {
+				await this.save(user, options);
+			}
+		}, 'UserModel::saveMulti');
+	}
+
 }

packages/server/src/routes/api/debug.ts

@@ -1,5 +1,5 @@
 import config from '../../config';
-import { createTestUsers } from '../../tools/debugTools';
+import { clearDatabase, createTestUsers, CreateTestUsersOptions } from '../../tools/debugTools';
 import { bodyFields } from '../../utils/requestUtils';
 import Router from '../../utils/Router';
 import { RouteType } from '../../utils/types';
@@ -12,6 +12,8 @@ router.public = true;
 
 interface Query {
 	action: string;
+	count?: number;
+	fromNum?: number;
 }
 
 router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
@@ -20,7 +22,16 @@ router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
 	console.info(`Action: ${query.action}`);
 
 	if (query.action === 'createTestUsers') {
-		await createTestUsers(ctx.joplin.db, config());
+		const options: CreateTestUsersOptions = {};
+
+		if ('count' in query) options.count = query.count;
+		if ('fromNum' in query) options.fromNum = query.fromNum;
+
+		await createTestUsers(ctx.joplin.db, config(), options);
+	}
+
+	if (query.action === 'clearDatabase') {
+		await clearDatabase(ctx.joplin.db);
 	}
 });
 

packages/server/src/tools/debugTools.ts

@@ -1,9 +1,14 @@
 import { DbConnection, dropTables, migrateLatest } from '../db';
 import newModelFactory from '../models/factory';
 import { AccountType } from '../models/UserModel';
-import { UserFlagType } from '../services/database/types';
+import { User, UserFlagType } from '../services/database/types';
 import { Config } from '../utils/types';
 
+export interface CreateTestUsersOptions {
+	count?: number;
+	fromNum?: number;
+}
+
 export async function handleDebugCommands(argv: any, db: DbConnection, config: Config): Promise<boolean> {
 	if (argv.debugCreateTestUsers) {
 		await createTestUsers(db, config);
@@ -14,51 +19,79 @@ export async function handleDebugCommands(argv: any, db: DbConnection, config: C
 	return true;
 }
 
-export async function createTestUsers(db: DbConnection, config: Config) {
+export async function clearDatabase(db: DbConnection) {
 	await dropTables(db);
 	await migrateLatest(db);
+}
+
+export async function createTestUsers(db: DbConnection, config: Config, options: CreateTestUsersOptions = null) {
+	options = {
+		count: 0,
+		fromNum: 1,
+		...options,
+	};
 
 	const password = 'hunter1hunter2hunter3';
-	const models = newModelFactory(db, config);
 
-	for (let userNum = 1; userNum <= 2; userNum++) {
-		await models.user().save({
-			email: `user${userNum}@example.com`,
-			password,
-			full_name: `User ${userNum}`,
-		});
-	}
+	if (options.count) {
+		const models = newModelFactory(db, config);
 
-	{
-		const { user } = await models.subscription().saveUserAndSubscription(
-			'usersub@example.com',
-			'With Sub',
-			AccountType.Basic,
-			'usr_111',
-			'sub_111'
-		);
-		await models.user().save({ id: user.id, password });
-	}
+		const users: User[] = [];
 
-	{
-		const { user, subscription } = await models.subscription().saveUserAndSubscription(
-			'userfailedpayment@example.com',
-			'Failed Payment',
-			AccountType.Basic,
-			'usr_222',
-			'sub_222'
-		);
-		await models.user().save({ id: user.id, password });
-		await models.subscription().handlePayment(subscription.stripe_subscription_id, false);
-	}
+		for (let i = 0; i < options.count; i++) {
+			const userNum = i + options.fromNum;
+			users.push({
+				email: `user${userNum}@example.com`,
+				password,
+				full_name: `User ${userNum}`,
+			});
+		}
 
-	{
-		const user = await models.user().save({
-			email: 'userwithflags@example.com',
-			password,
-			full_name: 'User Withflags',
-		});
+		await models.user().saveMulti(users);
+	} else {
+		await dropTables(db);
+		await migrateLatest(db);
+		const models = newModelFactory(db, config);
 
-		await models.userFlag().add(user.id, UserFlagType.AccountOverLimit);
+		for (let userNum = 1; userNum <= 2; userNum++) {
+			await models.user().save({
+				email: `user${userNum}@example.com`,
+				password,
+				full_name: `User ${userNum}`,
+			});
+		}
+
+		{
+			const { user } = await models.subscription().saveUserAndSubscription(
+				'usersub@example.com',
+				'With Sub',
+				AccountType.Basic,
+				'usr_111',
+				'sub_111'
+			);
+			await models.user().save({ id: user.id, password });
+		}
+
+		{
+			const { user, subscription } = await models.subscription().saveUserAndSubscription(
+				'userfailedpayment@example.com',
+				'Failed Payment',
+				AccountType.Basic,
+				'usr_222',
+				'sub_222'
+			);
+			await models.user().save({ id: user.id, password });
+			await models.subscription().handlePayment(subscription.stripe_subscription_id, false);
+		}
+
+		{
+			const user = await models.user().save({
+				email: 'userwithflags@example.com',
+				password,
+				full_name: 'User Withflags',
+			});
+
+			await models.userFlag().add(user.id, UserFlagType.AccountOverLimit);
+		}
 	}
 }