diff --git a/packages/tools/fuzzer/ActionTracker.ts b/packages/tools/fuzzer/ActionTracker.ts index 090258c5d1..82f331ff6e 100644 --- a/packages/tools/fuzzer/ActionTracker.ts +++ b/packages/tools/fuzzer/ActionTracker.ts @@ -56,10 +56,13 @@ class ActionTracker { public track(client: { email: string }) { const clientId = client.email; - this.tree_.set(clientId, { - childIds: [], - sharedFolderIds: [], - }); + // If the client's remote account already exists, continue using it: + if (!this.tree_.has(clientId)) { + this.tree_.set(clientId, { + childIds: [], + sharedFolderIds: [], + }); + } const getChildIds = (itemId: ItemId) => { const item = this.idToItem_.get(itemId); diff --git a/packages/tools/fuzzer/Client.ts b/packages/tools/fuzzer/Client.ts index a3f154de35..c05ef2b4fe 100644 --- a/packages/tools/fuzzer/Client.ts +++ b/packages/tools/fuzzer/Client.ts @@ -1,5 +1,5 @@ import uuid, { createSecureRandom } from '@joplin/lib/uuid'; -import { ActionableClient, FolderMetadata, FuzzContext, HttpMethod, ItemId, Json, NoteData, RandomFolderOptions, UserData } from './types'; +import { ActionableClient, FolderMetadata, FuzzContext, HttpMethod, ItemId, Json, NoteData, RandomFolderOptions } from './types'; import { join } from 'path'; import { mkdir, remove } from 'fs-extra'; import getStringProperty from './utils/getStringProperty'; @@ -12,6 +12,7 @@ import { commandToString } from '@joplin/utils'; import { quotePath } from '@joplin/utils/path'; import getNumberProperty from './utils/getNumberProperty'; import retryWithCount from './utils/retryWithCount'; +import resolvePathWithinDir from '@joplin/lib/utils/resolvePathWithinDir'; import { msleep, Second } from '@joplin/utils/time'; import shim from '@joplin/lib/shim'; import { spawn } from 'child_process'; @@ -21,6 +22,62 @@ import Stream = require('stream'); const logger = Logger.create('Client'); +type AccountData = Readonly<{ + email: string; + password: string; + serverId: string; + e2eePassword: string; + associatedClientCount: number; + onClientConnected: ()=> void; + onClientDisconnected: ()=> Promise; +}>; + +const createNewAccount = async (email: string, context: FuzzContext): Promise => { + const password = createSecureRandom(); + const apiOutput = await context.execApi('POST', 'api/users', { + email, + }); + const serverId = getStringProperty(apiOutput, 'id'); + + // The password needs to be set *after* creating the user. + const userRoute = `api/users/${encodeURIComponent(serverId)}`; + await context.execApi('PATCH', userRoute, { + email, + password, + email_confirmed: 1, + }); + + const closeAccount = async () => { + await context.execApi('DELETE', userRoute, {}); + }; + + let referenceCounter = 0; + return { + email, + password, + e2eePassword: createSecureRandom().replace(/^-/, '_'), + serverId, + get associatedClientCount() { + return referenceCounter; + }, + onClientConnected: () => { + referenceCounter++; + }, + onClientDisconnected: async () => { + referenceCounter --; + assert.ok(referenceCounter >= 0, 'reference counter should be non-negative'); + if (referenceCounter === 0) { + await closeAccount(); + } + }, + }; +}; + +type ApiData = Readonly<{ + port: number; + token: string; +}>; + type OnCloseListener = ()=> void; type ChildProcessWrapper = { @@ -33,76 +90,56 @@ type ChildProcessWrapper = { // Should match the prompt used by the CLI "batch" command. const cliProcessPromptString = 'command> '; - - class Client implements ActionableClient { public readonly email: string; public static async create(actionTracker: ActionTracker, context: FuzzContext) { + const account = await createNewAccount(`${uuid.create()}@localhost`, context); + + try { + return await this.fromAccount(account, actionTracker, context); + } catch (error) { + logger.error('Error creating client:', error); + await account.onClientDisconnected(); + throw error; + } + } + + private static async fromAccount(account: AccountData, actionTracker: ActionTracker, context: FuzzContext) { const id = uuid.create(); const profileDirectory = join(context.baseDir, id); await mkdir(profileDirectory); - const email = `${id}@localhost`; - const password = createSecureRandom(); - const apiOutput = await context.execApi('POST', 'api/users', { - email, - }); - const serverId = getStringProperty(apiOutput, 'id'); - - // The password needs to be set *after* creating the user. - const userRoute = `api/users/${encodeURIComponent(serverId)}`; - await context.execApi('PATCH', userRoute, { - email, - password, - email_confirmed: 1, - }); - - const closeAccount = async () => { - await context.execApi('DELETE', userRoute, {}); + const apiData: ApiData = { + token: createSecureRandom().replace(/[-]/g, '_'), + port: await ClipperServer.instance().findAvailablePort(), }; - try { - const userData = { - email: getStringProperty(apiOutput, 'email'), - password, - }; + const client = new Client( + context, + actionTracker, + actionTracker.track({ email: account.email }), + account, + profileDirectory, + apiData, + `${account.email}${account.associatedClientCount ? ` (${account.associatedClientCount})` : ''}`, + ); - assert.equal(email, userData.email); + account.onClientConnected(); - const apiToken = createSecureRandom().replace(/[-]/g, '_'); - const apiPort = await ClipperServer.instance().findAvailablePort(); + // Joplin Server sync + await client.execCliCommand_('config', 'sync.target', '9'); + await client.execCliCommand_('config', 'sync.9.path', context.serverUrl); + await client.execCliCommand_('config', 'sync.9.username', account.email); + await client.execCliCommand_('config', 'sync.9.password', account.password); + await client.execCliCommand_('config', 'api.token', apiData.token); + await client.execCliCommand_('config', 'api.port', String(apiData.port)); - const client = new Client( - actionTracker.track({ email }), - userData, - profileDirectory, - apiPort, - apiToken, - ); + await client.execCliCommand_('e2ee', 'enable', '--password', account.e2eePassword); + logger.info('Created and configured client'); - client.onClose(closeAccount); - - // Joplin Server sync - await client.execCliCommand_('config', 'sync.target', '9'); - await client.execCliCommand_('config', 'sync.9.path', context.serverUrl); - await client.execCliCommand_('config', 'sync.9.username', userData.email); - await client.execCliCommand_('config', 'sync.9.password', userData.password); - await client.execCliCommand_('config', 'api.token', apiToken); - await client.execCliCommand_('config', 'api.port', String(apiPort)); - - const e2eePassword = createSecureRandom().replace(/^-/, '_'); - await client.execCliCommand_('e2ee', 'enable', '--password', e2eePassword); - logger.info('Created and configured client'); - - await client.startClipperServer_(); - - await client.sync(); - return client; - } catch (error) { - await closeAccount(); - throw error; - } + await client.startClipperServer_(); + return client; } private onCloseListeners_: OnCloseListener[] = []; @@ -116,13 +153,15 @@ class Client implements ActionableClient { private transcript_: string[] = []; private constructor( + private readonly context_: FuzzContext, + private readonly globalActionTracker_: ActionTracker, private readonly tracker_: ActionableClient, - userData: UserData, + private readonly account_: AccountData, private readonly profileDirectory: string, - private readonly apiPort_: number, - private readonly apiToken_: string, + private readonly apiData_: ApiData, + private readonly clientLabel_: string, ) { - this.email = userData.email; + this.email = account_.email; // Don't skip child process-related tasks. this.childProcessQueue_.setCanSkipTaskHandler(() => false); @@ -186,9 +225,11 @@ class Client implements ActionableClient { public async close() { assert.ok(!this.closed_, 'should not be closed'); + await this.account_.onClientDisconnected(); + // Before removing the profile directory, verify that the profile directory is in the // expected location: - const profileDirectory = this.profileDirectory; + const profileDirectory = resolvePathWithinDir(this.context_.baseDir, this.profileDirectory); assert.ok(profileDirectory, 'profile directory for client should be contained within the main temporary profiles directory (should be safe to delete)'); await remove(profileDirectory); @@ -204,8 +245,16 @@ class Client implements ActionableClient { this.onCloseListeners_.push(listener); } + public async createClientOnSameAccount() { + return await Client.fromAccount(this.account_, this.globalActionTracker_, this.context_); + } + + public hasSameAccount(other: Client) { + return other.account_ === this.account_; + } + public get label() { - return this.email; + return this.clientLabel_; } private get cliCommandArguments() { @@ -318,8 +367,8 @@ class Client implements ActionableClient { // eslint-disable-next-line no-dupe-class-members -- This is not a duplicate class member private async execApiCommand_(method: HttpMethod, route: string, data: Json|null = null): Promise { route = route.replace(/^[/]/, ''); - const url = new URL(`http://localhost:${this.apiPort_}/${route}`); - url.searchParams.append('token', this.apiToken_); + const url = new URL(`http://localhost:${this.apiData_.port}/${route}`); + url.searchParams.append('token', this.apiData_.token); this.transcript_.push(`\n[[${method} ${url}; body: ${JSON.stringify(data)}]]\n`); diff --git a/packages/tools/fuzzer/ClientPool.ts b/packages/tools/fuzzer/ClientPool.ts index 65c40a5ab5..c0244dfa70 100644 --- a/packages/tools/fuzzer/ClientPool.ts +++ b/packages/tools/fuzzer/ClientPool.ts @@ -45,6 +45,17 @@ export default class ClientPool { ]; } + public async newWithSameAccount(sourceClient: Client) { + const client = await sourceClient.createClientOnSameAccount(); + this.listenForClientClose_(client); + this.clients_ = [...this.clients_, client]; + return client; + } + + public othersWithSameAccount(client: Client) { + return this.clients_.filter(other => other !== client && other.hasSameAccount(client)); + } + public async checkState() { for (const client of this.clients_) { await client.checkState(); diff --git a/packages/tools/fuzzer/sync-fuzzer.ts b/packages/tools/fuzzer/sync-fuzzer.ts index 1b4b6a3974..1939d1242c 100644 --- a/packages/tools/fuzzer/sync-fuzzer.ts +++ b/packages/tools/fuzzer/sync-fuzzer.ts @@ -150,7 +150,7 @@ const doRandomAction = async (context: FuzzContext, client: Client, clientPool: }); if (!target) return false; - const other = clientPool.randomClient(c => c !== client); + const other = clientPool.randomClient(c => !c.hasSameAccount(client)); await client.shareFolder(target.id, other); return true; }, @@ -191,6 +191,63 @@ const doRandomAction = async (context: FuzzContext, client: Client, clientPool: await client.moveItem(target.id, newParent.id); return true; }, + newClientOnSameAccount: async () => { + const welcomeNoteCount = context.randInt(0, 30); + logger.info(`Syncing a new client on the same account ${welcomeNoteCount > 0 ? `(with ${welcomeNoteCount} initial notes)` : ''}`); + const createClientInitialNotes = async (client: Client) => { + if (welcomeNoteCount === 0) return; + + // Create a new folder. Usually, new clients have a default set of + // welcome notes when first syncing. + const testNotesFolderId = uuid.create(); + await client.createFolder({ + id: testNotesFolderId, + title: 'Test -- from secondary client', + parentId: '', + }); + + for (let i = 0; i < welcomeNoteCount; i++) { + await client.createNote({ + parentId: testNotesFolderId, + id: uuid.create(), + title: `Test note ${i}/${welcomeNoteCount}`, + body: `Test note (in account ${client.email}), created ${Date.now()}.`, + }); + } + }; + + await client.sync(); + + const other = await clientPool.newWithSameAccount(client); + await createClientInitialNotes(other); + + // Sometimes, a delay is needed between client creation + // and initial sync. Retry the initial sync and the checkState + // on failure: + await retryWithCount(async () => { + await other.sync(); + await other.checkState(); + }, { + delayOnFailure: (count) => Second * count, + count: 3, + onFail: async (error) => { + logger.warn('other.sync/other.checkState failed with', error, 'retrying...'); + }, + }); + + await client.sync(); + return true; + }, + removeClientsOnSameAccount: async () => { + const others = clientPool.othersWithSameAccount(client); + if (others.length === 0) return false; + + for (const otherClient of others) { + assert.notEqual(otherClient, client); + await otherClient.close(); + } + return true; + }, }; const actionKeys = [...Object.keys(actions)] as (keyof typeof actions)[];