self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-03-03 15:32:30 +02:00
Code Issues Releases Activity

Tools: Added tools to stress test Joplin Server

Browse Source
This commit is contained in:
Laurent Cozic 2021-09-17 10:59:10 +01:00
parent 4e70ca6fd0
commit b56177a4e3
12 changed files with 379 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.eslintignore
View File

@ -76,6 +76,9 @@ packages/app-cli/app/command-e2ee.js.map
packages/app-cli/app/command-settingschema.d.ts
packages/app-cli/app/command-settingschema.js
packages/app-cli/app/command-settingschema.js.map
packages/app-cli/app/command-testing.d.ts
packages/app-cli/app/command-testing.js
packages/app-cli/app/command-testing.js.map
packages/app-cli/app/services/plugins/PluginRunner.d.ts
packages/app-cli/app/services/plugins/PluginRunner.js
packages/app-cli/app/services/plugins/PluginRunner.js.map
@ -109,6 +112,9 @@ packages/app-cli/tests/services/plugins/sandboxProxy.js.map
packages/app-cli/tests/testUtils.d.ts
packages/app-cli/tests/testUtils.js
packages/app-cli/tests/testUtils.js.map
packages/app-cli/tools/populateDatabase.d.ts
packages/app-cli/tools/populateDatabase.js
packages/app-cli/tools/populateDatabase.js.map
packages/app-desktop/ElectronAppWrapper.d.ts
packages/app-desktop/ElectronAppWrapper.js
packages/app-desktop/ElectronAppWrapper.js.map

6
.gitignore vendored
View File

@ -61,6 +61,9 @@ packages/app-cli/app/command-e2ee.js.map
packages/app-cli/app/command-settingschema.d.ts
packages/app-cli/app/command-settingschema.js
packages/app-cli/app/command-settingschema.js.map
packages/app-cli/app/command-testing.d.ts
packages/app-cli/app/command-testing.js
packages/app-cli/app/command-testing.js.map
packages/app-cli/app/services/plugins/PluginRunner.d.ts
packages/app-cli/app/services/plugins/PluginRunner.js
packages/app-cli/app/services/plugins/PluginRunner.js.map
@ -94,6 +97,9 @@ packages/app-cli/tests/services/plugins/sandboxProxy.js.map
packages/app-cli/tests/testUtils.d.ts
packages/app-cli/tests/testUtils.js
packages/app-cli/tests/testUtils.js.map
packages/app-cli/tools/populateDatabase.d.ts
packages/app-cli/tools/populateDatabase.js
packages/app-cli/tools/populateDatabase.js.map
packages/app-desktop/ElectronAppWrapper.d.ts
packages/app-desktop/ElectronAppWrapper.js
packages/app-desktop/ElectronAppWrapper.js.map

4
packages/app-cli/.gitignore vendored
View File

@ -23,4 +23,6 @@ tests/support/dropbox-auth.txt
tests/support/nextcloud-auth.json
tests/support/onedrive-auth.txt
build/
patches/
patches/
createUsers-*.txt
tools/temp/

2
packages/app-cli/app/cli-utils.js
View File

@ -89,7 +89,7 @@ cliUtils.makeCommandArgs = function(cmd, argv) {
flags = cliUtils.parseFlags(flags);
if (!flags.arg) {
booleanFlags.push(flags.short);
if (flags.short) booleanFlags.push(flags.short);
if (flags.long) booleanFlags.push(flags.long);
}

95
packages/app-cli/app/command-testing.ts Normal file
View File

@ -0,0 +1,95 @@
const { BaseCommand } = require('./base-command.js');
import { reg } from '@joplin/lib/registry';
import Note from '@joplin/lib/models/Note';
import uuid from '@joplin/lib/uuid';
import populateDatabase from '@joplin/lib/services/debug/populateDatabase';
function randomElement(array: any[]): any {
if (!array.length) return null;
return array[Math.floor(Math.random() * array.length)];
}
function itemCount(args: any) {
const count = Number(args.arg0);
if (!count || isNaN(count)) throw new Error('Note count must be specified');
return count;
}
class Command extends BaseCommand {
usage() {
return 'testing <command> [arg0]';
}
description() {
return 'testing';
}
enabled() {
return false;
}
options(): any[] {
return [
['--folder-count <count>', 'Folders to create'],
['--note-count <count>', 'Notes to create'],
['--tag-count <count>', 'Tags to create'],
['--tags-per-note <count>', 'Tags per note'],
['--silent', 'Silent'],
];
}
async action(args: any) {
const { command, options } = args;
if (command === 'populate') {
await populateDatabase(reg.db(), {
folderCount: options['folder-count'],
noteCount: options['note-count'],
tagCount: options['tag-count'],
tagsPerNote: options['tags-per-note'],
silent: options['silent'],
});
}
const promises: any[] = [];
if (command === 'createRandomNotes') {
const noteCount = itemCount(args);
for (let i = 0; i < noteCount; i++) {
promises.push(Note.save({
title: `Note ${uuid.createNano()}`,
}));
}
}
if (command === 'updateRandomNotes') {
const noteCount = itemCount(args);
const noteIds = await Note.allIds();
for (let i = 0; i < noteCount; i++) {
const noteId = randomElement(noteIds);
promises.push(Note.save({
id: noteId,
title: `Note ${uuid.createNano()}`,
}));
}
}
if (command === 'deleteRandomNotes') {
const noteCount = itemCount(args);
const noteIds = await Note.allIds();
for (let i = 0; i < noteCount; i++) {
const noteId = randomElement(noteIds);
promises.push(Note.delete(noteId));
}
}
await Promise.all(promises);
}
}
module.exports = Command;

52
packages/app-cli/createUsers.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/bash
# Start the server with:
#
# JOPLIN_IS_TESTING=1 npm run start-dev
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
# curl --data '{"action": "clearDatabase"}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
# SMALL
# curl --data '{"action": "createTestUsers", "count": 400, "fromNum": 1}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
NUM=398
while [ "$NUM" -lt 400 ]; do
NUM=$(( NUM + 1 ))
echo "User $NUM"
CMD_FILE="$SCRIPT_DIR/createUsers-$NUM.txt"
PROFILE_DIR=~/.config/joplindev-testing-$NUM
USER_EMAIL="user$NUM@example.com"
rm -rf "$CMD_FILE" "$PROFILE_DIR"
touch "$CMD_FILE"
FLAG_FOLDER_COUNT=100
FLAG_NOTE_COUNT=1000
FLAG_TAG_COUNT=20
if [ "$NUM" -gt 300 ]; then
FLAG_FOLDER_COUNT=2000
FLAG_NOTE_COUNT=10000
FLAG_TAG_COUNT=200
fi
if [ "$NUM" -gt 399 ]; then
FLAG_FOLDER_COUNT=10000
FLAG_NOTE_COUNT=150000
FLAG_TAG_COUNT=2000
fi
echo "testing populate --silent --folder-count $FLAG_FOLDER_COUNT --note-count $FLAG_NOTE_COUNT --tag-count $FLAG_TAG_COUNT" >> "$CMD_FILE"
echo "config keychain.supported 0" >> "$CMD_FILE"
echo "config sync.target 10" >> "$CMD_FILE"
echo "config sync.10.username $USER_EMAIL" >> "$CMD_FILE"
echo "config sync.10.password hunter1hunter2hunter3" >> "$CMD_FILE"
echo "sync" >> "$CMD_FILE"
npm start -- --profile "$PROFILE_DIR" batch "$CMD_FILE"
done

1
packages/app-cli/package.json
View File

@ -10,6 +10,7 @@
"test-ci": "jest --config=jest.config.js --forceExit",
"build": "gulp build",
"start": "gulp build -L && node \"build/main.js\" --stack-trace-enabled --log-level debug --env dev",
"start-no-build": "node \"build/main.js\" --stack-trace-enabled --log-level debug --env dev",
"tsc": "node node_modules/typescript/bin/tsc --project tsconfig.json",
"watch": "node node_modules/typescript/bin/tsc --watch --project tsconfig.json"
},

86
packages/app-cli/tools/populateDatabase.ts Normal file
View File

@ -0,0 +1,86 @@
import * as fs from 'fs-extra';
import { homedir } from 'os';
import { execCommand2 } from '@joplin/tools/tool-utils';
import { chdir } from 'process';
const minUserNum = 1;
const maxUserNum = 400;
const cliDir = `${__dirname}/..`;
const tempDir = `${__dirname}/temp`;
function randomInt(min: number, max: number) {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min + 1)) + min;
}
const processing_: Record<number, boolean> = {};
const processUser = async (userNum: number) => {
if (processing_[userNum]) {
console.info(`User already being processed: ${userNum} - skipping`);
return;
}
processing_[userNum] = true;
try {
const userEmail = `user${userNum}@example.com`;
const userPassword = 'hunter1hunter2hunter3';
const commandFile = `${tempDir}/populateDatabase-${userNum}.txt`;
const profileDir = `${homedir()}/.config/joplindev-populate/joplindev-testing-${userNum}`;
const commands: string[] = [];
const jackpot = Math.random() >= 0.95 ? 100 : 1;
commands.push(`testing createRandomNotes ${randomInt(1, 500 * jackpot)}`);
commands.push(`testing updateRandomNotes ${randomInt(1, 1500 * jackpot)}`);
commands.push(`testing deleteRandomNotes ${randomInt(1, 200 * jackpot)}`);
commands.push('config keychain.supported 0');
commands.push('config sync.target 10');
commands.push(`config sync.10.username ${userEmail}`);
commands.push(`config sync.10.password ${userPassword}`);
commands.push('sync');
await fs.writeFile(commandFile, commands.join('\n'), 'utf8');
await chdir(cliDir);
await execCommand2(['npm', 'run', 'start-no-build', '--', '--profile', profileDir, 'batch', commandFile]);
} catch (error) {
console.error(`Could not process user ${userNum}:`, error);
} finally {
delete processing_[userNum];
}
};
const waitForProcessing = (count: number) => {
return new Promise((resolve) => {
const iid = setInterval(() => {
if (Object.keys(processing_).length <= count) {
clearInterval(iid);
resolve(null);
}
}, 100);
});
};
const main = async () => {
await fs.mkdirp(tempDir);
// Build the app once before starting, because we'll use start-no-build to
// run the scripts (faster)
await execCommand2(['npm', 'run', 'build']);
while (true) {
const userNum = randomInt(minUserNum, maxUserNum);
void processUser(userNum);
await waitForProcessing(10);
}
};
main().catch((error) => {
console.error('Fatal error', error);
process.exit(1);
});

59
packages/lib/services/debug/populateDatabase.ts
View File

File diff suppressed because one or more lines are too long

8
packages/server/src/models/UserModel.ts
View File

@ -469,4 +469,12 @@ export default class UserModel extends BaseModel<User> {
}, 'UserModel::save');
}
public async saveMulti(users: User[], options: SaveOptions = {}): Promise<void> {
await this.withTransaction(async () => {
for (const user of users) {
await this.save(user, options);
}
}, 'UserModel::saveMulti');
}
}

15
packages/server/src/routes/api/debug.ts
View File

@ -1,5 +1,5 @@
import config from '../../config';
import { createTestUsers } from '../../tools/debugTools';
import { clearDatabase, createTestUsers, CreateTestUsersOptions } from '../../tools/debugTools';
import { bodyFields } from '../../utils/requestUtils';
import Router from '../../utils/Router';
import { RouteType } from '../../utils/types';
@ -12,6 +12,8 @@ router.public = true;
interface Query {
action: string;
count?: number;
fromNum?: number;
}
router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
@ -20,7 +22,16 @@ router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
console.info(`Action: ${query.action}`);
if (query.action === 'createTestUsers') {
await createTestUsers(ctx.joplin.db, config());
const options: CreateTestUsersOptions = {};
if ('count' in query) options.count = query.count;
if ('fromNum' in query) options.fromNum = query.fromNum;
await createTestUsers(ctx.joplin.db, config(), options);
}
if (query.action === 'clearDatabase') {
await clearDatabase(ctx.joplin.db);
}
});

109
packages/server/src/tools/debugTools.ts
View File

@ -1,9 +1,14 @@
import { DbConnection, dropTables, migrateLatest } from '../db';
import newModelFactory from '../models/factory';
import { AccountType } from '../models/UserModel';
import { UserFlagType } from '../services/database/types';
import { User, UserFlagType } from '../services/database/types';
import { Config } from '../utils/types';
export interface CreateTestUsersOptions {
count?: number;
fromNum?: number;
}
export async function handleDebugCommands(argv: any, db: DbConnection, config: Config): Promise<boolean> {
if (argv.debugCreateTestUsers) {
await createTestUsers(db, config);
@ -14,51 +19,79 @@ export async function handleDebugCommands(argv: any, db: DbConnection, config: C
return true;
}
export async function createTestUsers(db: DbConnection, config: Config) {
export async function clearDatabase(db: DbConnection) {
await dropTables(db);
await migrateLatest(db);
}
export async function createTestUsers(db: DbConnection, config: Config, options: CreateTestUsersOptions = null) {
options = {
count: 0,
fromNum: 1,
...options,
};
const password = 'hunter1hunter2hunter3';
const models = newModelFactory(db, config);
for (let userNum = 1; userNum <= 2; userNum++) {
await models.user().save({
email: `user${userNum}@example.com`,
password,
full_name: `User ${userNum}`,
});
}
if (options.count) {
const models = newModelFactory(db, config);
{
const { user } = await models.subscription().saveUserAndSubscription(
'usersub@example.com',
'With Sub',
AccountType.Basic,
'usr_111',
'sub_111'
);
await models.user().save({ id: user.id, password });
}
const users: User[] = [];
{
const { user, subscription } = await models.subscription().saveUserAndSubscription(
'userfailedpayment@example.com',
'Failed Payment',
AccountType.Basic,
'usr_222',
'sub_222'
);
await models.user().save({ id: user.id, password });
await models.subscription().handlePayment(subscription.stripe_subscription_id, false);
}
for (let i = 0; i < options.count; i++) {
const userNum = i + options.fromNum;
users.push({
email: `user${userNum}@example.com`,
password,
full_name: `User ${userNum}`,
});
}
{
const user = await models.user().save({
email: 'userwithflags@example.com',
password,
full_name: 'User Withflags',
});
await models.user().saveMulti(users);
} else {
await dropTables(db);
await migrateLatest(db);
const models = newModelFactory(db, config);
await models.userFlag().add(user.id, UserFlagType.AccountOverLimit);
for (let userNum = 1; userNum <= 2; userNum++) {
await models.user().save({
email: `user${userNum}@example.com`,
password,
full_name: `User ${userNum}`,
});
}
{
const { user } = await models.subscription().saveUserAndSubscription(
'usersub@example.com',
'With Sub',
AccountType.Basic,
'usr_111',
'sub_111'
);
await models.user().save({ id: user.id, password });
}
{
const { user, subscription } = await models.subscription().saveUserAndSubscription(
'userfailedpayment@example.com',
'Failed Payment',
AccountType.Basic,
'usr_222',
'sub_222'
);
await models.user().save({ id: user.id, password });
await models.subscription().handlePayment(subscription.stripe_subscription_id, false);
}
{
const user = await models.user().save({
email: 'userwithflags@example.com',
password,
full_name: 'User Withflags',
});
await models.userFlag().add(user.id, UserFlagType.AccountOverLimit);
}
}
}