Chore: Sync fuzzer: Allow generating large amounts of test data for Joplin Server (#13636)

Co-authored-by: Laurent Cozic <laurent22@users.noreply.github.com>

.eslintignore

@@ -1827,14 +1827,18 @@ packages/tools/fuzzer/Client.js
 packages/tools/fuzzer/ClientPool.js
 packages/tools/fuzzer/Server.js
 packages/tools/fuzzer/constants.js
+packages/tools/fuzzer/doRandomAction.js
 packages/tools/fuzzer/model/FolderRecord.js
 packages/tools/fuzzer/sync-fuzzer.js
 packages/tools/fuzzer/types.js
+packages/tools/fuzzer/utils/ProgressBar.js
 packages/tools/fuzzer/utils/SeededRandom.js
 packages/tools/fuzzer/utils/getNumberProperty.js
 packages/tools/fuzzer/utils/getProperty.js
 packages/tools/fuzzer/utils/getStringProperty.js
+packages/tools/fuzzer/utils/logDiffDebug.js
 packages/tools/fuzzer/utils/openDebugSession.js
+packages/tools/fuzzer/utils/randomString.js
 packages/tools/fuzzer/utils/retryWithCount.js
 packages/tools/generate-database-types.js
 packages/tools/generate-images.js

.gitignore

@@ -1799,14 +1799,18 @@ packages/tools/fuzzer/Client.js
 packages/tools/fuzzer/ClientPool.js
 packages/tools/fuzzer/Server.js
 packages/tools/fuzzer/constants.js
+packages/tools/fuzzer/doRandomAction.js
 packages/tools/fuzzer/model/FolderRecord.js
 packages/tools/fuzzer/sync-fuzzer.js
 packages/tools/fuzzer/types.js
+packages/tools/fuzzer/utils/ProgressBar.js
 packages/tools/fuzzer/utils/SeededRandom.js
 packages/tools/fuzzer/utils/getNumberProperty.js
 packages/tools/fuzzer/utils/getProperty.js
 packages/tools/fuzzer/utils/getStringProperty.js
+packages/tools/fuzzer/utils/logDiffDebug.js
 packages/tools/fuzzer/utils/openDebugSession.js
+packages/tools/fuzzer/utils/randomString.js
 packages/tools/fuzzer/utils/retryWithCount.js
 packages/tools/generate-database-types.js
 packages/tools/generate-images.js

packages/tools/fuzzer/ActionTracker.ts

@@ -518,8 +518,7 @@ class ActionTracker {
 					folders = folders.filter(folder => !isReadOnly(folder.id));
 				}
 
-				const folderIndex = this.context_.randInt(0, folders.length);
+				return folders.length ? this.context_.randomFrom(folders) : null;
-				return folders.length ? folders[folderIndex] : null;
 			},
 			randomNote: async () => {
 				const notes = await tracker.listNotes();

packages/tools/fuzzer/Client.ts

@@ -13,12 +13,15 @@ import { quotePath } from '@joplin/utils/path';
 import getNumberProperty from './utils/getNumberProperty';
 import retryWithCount from './utils/retryWithCount';
 import resolvePathWithinDir from '@joplin/lib/utils/resolvePathWithinDir';
-import { msleep, Second } from '@joplin/utils/time';
+import { formatMsToDateTimeLocal, msleep, Second } from '@joplin/utils/time';
 import shim from '@joplin/lib/shim';
 import { spawn } from 'child_process';
 import AsyncActionQueue from '@joplin/lib/AsyncActionQueue';
 import { createInterface } from 'readline/promises';
 import Stream = require('stream');
+import ProgressBar from './utils/ProgressBar';
+import logDiffDebug from './utils/logDiffDebug';
+import { NoteEntity } from '@joplin/lib/services/database/types';
 
 const logger = Logger.create('Client');
 
@@ -26,7 +29,7 @@ type AccountData = Readonly<{
 	email: string;
 	password: string;
 	serverId: string;
-	e2eePassword: string;
+	e2eePassword: string|null;
 	associatedClientCount: number;
 	onClientConnected: ()=> void;
 	onClientDisconnected: ()=> Promise<void>;
@@ -36,6 +39,7 @@ const createNewAccount = async (email: string, context: FuzzContext): Promise<Ac
 	const password = createSecureRandom();
 	const apiOutput = await context.execApi('POST', 'api/users', {
 		email,
+		full_name: `Fuzzer user from ${formatMsToDateTimeLocal(Date.now())}`,
 	});
 	const serverId = getStringProperty(apiOutput, 'id');
 
@@ -55,7 +59,7 @@ const createNewAccount = async (email: string, context: FuzzContext): Promise<Ac
 	return {
 		email,
 		password,
-		e2eePassword: createSecureRandom().replace(/^-/, '_'),
+		e2eePassword: context.enableE2ee ? createSecureRandom().replace(/^-/, '_') : null,
 		serverId,
 		get associatedClientCount() {
 			return referenceCounter;
@@ -66,7 +70,7 @@ const createNewAccount = async (email: string, context: FuzzContext): Promise<Ac
 		onClientDisconnected: async () => {
 			referenceCounter --;
 			assert.ok(referenceCounter >= 0, 'reference counter should be non-negative');
-			if (referenceCounter === 0) {
+			if (referenceCounter === 0 && !context.keepAccounts) {
 				await closeAccount();
 			}
 		},
@@ -90,6 +94,10 @@ type ChildProcessWrapper = {
 // Should match the prompt used by the CLI "batch" command.
 const cliProcessPromptString = 'command> ';
 
+interface CreateOrUpdateOptions {
+	quiet?: boolean;
+}
+
 class Client implements ActionableClient {
 	public readonly email: string;
 
@@ -97,7 +105,8 @@ class Client implements ActionableClient {
 		const account = await createNewAccount(`${uuid.create()}@localhost`, context);
 
 		try {
-			return await this.fromAccount(account, actionTracker, context);
+			const client = await this.fromAccount(account, actionTracker, context);
+			return client;
 		} catch (error) {
 			logger.error('Error creating client:', error);
 			await account.onClientDisconnected();
@@ -136,7 +145,9 @@ class Client implements ActionableClient {
 		await client.execCliCommand_('config', 'api.token', apiData.token);
 		await client.execCliCommand_('config', 'api.port', String(apiData.port));
 
+		if (account.e2eePassword) {
 			await client.execCliCommand_('e2ee', 'enable', '--password', account.e2eePassword);
+		}
 		logger.info('Created and configured client');
 
 		await client.startClipperServer_();
@@ -420,6 +431,8 @@ class Client implements ActionableClient {
 	}
 
 	private async decrypt_() {
+		if (!this.context_.enableE2ee) return;
+
 		const result = await this.execCliCommand_('e2ee', 'decrypt', '--force');
 		if (!result.stdout.includes('Completed decryption.')) {
 			throw new Error(`Decryption did not complete: ${result.stdout}`);
@@ -449,8 +462,75 @@ class Client implements ActionableClient {
 		});
 	}
 
-	public async createFolder(folder: FolderData) {
+	public async createOrUpdateMany(actionCount: number) {
+		logger.info(`Creating/updating ${actionCount} items...`);
+		const bar = new ProgressBar('Creating/updating');
+
+		const actions = {
+			create: async () => {
+				let parentId = (await this.randomFolder({ includeReadOnly: false }))?.id;
+				const createSubfolder = this.context_.randInt(0, 100) < 10;
+				if (!parentId || createSubfolder) {
+					const folder = await this.createRandomFolder(parentId, { quiet: true });
+					parentId = folder.id;
+				}
+
+				await this.createRandomNote(parentId, { quiet: true });
+			},
+			update: async (targetNote: NoteData) => {
+				const keep = targetNote.body.substring(
+					// Problems start to appear when notes get long.
+					// See https://github.com/laurent22/joplin/issues/13644.
+					0, Math.min(this.context_.randInt(0, targetNote.body.length), 5000),
+				);
+				const append = this.context_.randomString(this.context_.randInt(0, 5000));
+				await this.updateNote({
+					...targetNote,
+					body: keep + append,
+				}, { quiet: true });
+			},
+			delete: async (targetNote: NoteData) => {
+				await this.deleteNote(targetNote.id, { quiet: true });
+			},
+		};
+
+		for (let i = 0; i < actionCount; i++) {
+			bar.update(i, actionCount);
+
+			const actionId = this.context_.randInt(0, 100);
+
+			const targetNote = await this.randomNote({ includeReadOnly: false });
+			if (!targetNote) {
+				await actions.create();
+			} else if (actionId > 60) {
+				await actions.update(targetNote);
+			} else if (actionId > 50) {
+				await actions.delete(targetNote);
+			} else {
+				await actions.create();
+			}
+		}
+		bar.complete();
+	}
+
+	public async createRandomFolder(parentId: ItemId, options: CreateOrUpdateOptions) {
+		const titleLength = this.context_.randInt(1, 128);
+		const folderId = uuid.create();
+		const folder = {
+			parentId: parentId,
+			id: folderId,
+			title: this.context_.randomString(titleLength).replace(/\n/g, ' '),
+		};
+
+		await this.createFolder(folder, options);
+
+		return folder;
+	}
+
+	public async createFolder(folder: FolderData, { quiet = false }: CreateOrUpdateOptions = {}) {
+		if (!quiet) {
 			logger.info('Create folder', folder.id, 'in', `${folder.parentId ?? 'root'}/${this.label}`);
+		}
 		await this.tracker_.createFolder(folder);
 
 		await this.execApiCommand_('POST', '/folders', {
@@ -461,14 +541,32 @@ class Client implements ActionableClient {
 	}
 
 	private async assertNoteMatchesState_(expected: NoteData) {
+		const normalizeForCompare = (text: string) => {
+			// Handle invalid unicode (replace with placeholder characters)
+			return Buffer.from(new TextEncoder().encode(text))
+				.toString()
+				// Rule out differences caused by control characters:
+				.replace(/\p{C}/ug, '')
+				.trimEnd();
+		};
+
+		let lastActualNote: NoteEntity|null = null;
+		try {
 			await retryWithCount(async () => {
-			const noteContent = (await this.execCliCommand_('cat', expected.id)).stdout;
+				const noteResult = JSON.parse(
+					await this.execApiCommand_('GET', `/notes/${encodeURIComponent(expected.id)}?fields=title,body`),
+				);
+				lastActualNote = noteResult;
+
 				assert.equal(
-				// Compare without trailing newlines for consistency, the output from "cat"
+					normalizeForCompare(noteResult.title),
-				// can sometimes have an extra newline (due to the CLI prompt)
+					normalizeForCompare(expected.title),
-				noteContent.trimEnd(),
+					'note title should match',
-				`${expected.title}\n\n${expected.body.trimEnd()}`,
+				);
-				'note should exist',
+				assert.equal(
+					normalizeForCompare(noteResult.body),
+					normalizeForCompare(expected.body),
+					'note body should match',
 				);
 			}, {
 				count: 3,
@@ -478,10 +576,32 @@ class Client implements ActionableClient {
 					await this.execApiCommand_('GET', '/ping');
 				},
 			});
+		} catch (error) {
+			// Log additional information to help debug binary differences
+			if (lastActualNote) {
+				logDiffDebug(lastActualNote.title, expected.title);
+				logDiffDebug(lastActualNote.body, expected.body);
+			}
+			throw error;
+		}
 	}
 
-	public async createNote(note: NoteData) {
+	public async createRandomNote(parentId: string, { quiet = false }: CreateOrUpdateOptions = { }) {
+		const titleLength = this.context_.randInt(0, 256);
+		const bodyLength = this.context_.randInt(0, 2000);
+		await this.createNote({
+			published: false,
+			parentId,
+			title: this.context_.randomString(titleLength),
+			body: this.context_.randomString(bodyLength),
+			id: uuid.create(),
+		}, { quiet });
+	}
+
+	public async createNote(note: NoteData, { quiet = false }: CreateOrUpdateOptions = { }) {
+		if (!quiet) {
 			logger.info('Create note', note.id, 'in', `${note.parentId}/${this.label}`);
+		}
 		await this.tracker_.createNote(note);
 
 		await this.execApiCommand_('POST', '/notes', {
@@ -493,8 +613,11 @@ class Client implements ActionableClient {
 		await this.assertNoteMatchesState_(note);
 	}
 
-	public async updateNote(note: NoteData) {
+	public async updateNote(note: NoteData, { quiet = false }: CreateOrUpdateOptions = { }) {
+		if (!quiet) {
 			logger.info('Update note', note.id, 'in', `${note.parentId}/${this.label}`);
+		}
+
 		await this.tracker_.updateNote(note);
 		await this.execApiCommand_('PUT', `/notes/${encodeURIComponent(note.id)}`, {
 			title: note.title,
@@ -504,8 +627,11 @@ class Client implements ActionableClient {
 		await this.assertNoteMatchesState_(note);
 	}
 
-	public async deleteNote(id: ItemId) {
+	public async deleteNote(id: ItemId, { quiet }: CreateOrUpdateOptions = {}) {
+		if (!quiet) {
 			logger.info('Delete note', id, 'in', this.label);
+		}
+
 		await this.tracker_.deleteNote(id);
 
 		await this.execCliCommand_('rmnote', '--permanent', '--force', id);

packages/tools/fuzzer/ClientPool.ts

@@ -1,3 +1,4 @@
+import Logger from '@joplin/utils/Logger';
 import ActionTracker from './ActionTracker';
 import Client from './Client';
 import { CleanupTask, FuzzContext } from './types';
@@ -5,6 +6,8 @@ import { CleanupTask, FuzzContext } from './types';
 type AddCleanupTask = (task: CleanupTask)=> void;
 type ClientFilter = (client: Client)=> boolean;
 
+const logger = Logger.create('ClientPool');
+
 export default class ClientPool {
 	public static async create(
 		context: FuzzContext,
@@ -23,7 +26,7 @@ export default class ClientPool {
 
 		return new ClientPool(context, clientPool);
 	}
-	public constructor(
+	private constructor(
 		private readonly context_: FuzzContext,
 		private clients_: Client[],
 	) {
@@ -38,6 +41,16 @@ export default class ClientPool {
 		});
 	}
 
+	public async createInitialItemsAndSync() {
+		for (const client of this.clients) {
+			logger.info('Creating items for ', client.email);
+			const actionCount = this.context_.randomFrom([0, 10, 100]);
+			await client.createOrUpdateMany(actionCount);
+
+			await client.sync();
+		}
+	}
+
 	public clientsByEmail(email: string) {
 		return this.clients.filter(client => client.email === email);
 	}
@@ -67,9 +80,12 @@ export default class ClientPool {
 	}
 
 	public async syncAll() {
-		for (const client of this.clients_) {
+		// Sync all clients at roughly the same time. Some sync bugs are only apparent
-			await client.sync();
+		// when multiple clients are syncing simultaneously.
-		}
+		await Promise.all(this.clients_.map(c => c.sync()));
+
+		// Note: For more deterministic behavior, sync clients individually instead:
+		//   for (const client of this.clients_) { await client.sync(); }
 	}
 
 	public get clients() {

packages/tools/fuzzer/Server.ts

@@ -39,11 +39,15 @@ export default class Server {
 			cwd: serverDir,
 			stdin: 'ignore', // No stdin
 			// For debugging:
-			// stderr: process.stderr,
+			stderr: process.stderr,
 			// stdout: process.stdout,
 		});
 	}
 
+	public get url() {
+		return this.serverUrl_;
+	}
+
 	public async checkConnection() {
 		let lastError;
 		for (let retry = 0; retry < 30; retry++) {

packages/tools/fuzzer/doRandomAction.ts

@@ -0,0 +1,271 @@
+import uuid from '@joplin/lib/uuid';
+import Client from './Client';
+import ClientPool from './ClientPool';
+import { FuzzContext } from './types';
+import { strict as assert } from 'assert';
+import Logger from '@joplin/utils/Logger';
+import retryWithCount from './utils/retryWithCount';
+import { Second } from '@joplin/utils/time';
+
+const logger = Logger.create('doRandomAction');
+
+const doRandomAction = async (context: FuzzContext, client: Client, clientPool: ClientPool) => {
+	const selectOrCreateParentFolder = async () => {
+		let parentId = (await client.randomFolder({ includeReadOnly: false }))?.id;
+
+		// Create a toplevel folder to serve as this
+		// folder's parent if none exist yet
+		if (!parentId) {
+			parentId = uuid.create();
+			await client.createFolder({
+				parentId: '',
+				id: parentId,
+				title: 'Parent folder',
+			});
+		}
+
+		return parentId;
+	};
+
+	const defaultNoteProperties = {
+		published: false,
+	};
+
+	const selectOrCreateWriteableNote = async () => {
+		const options = { includeReadOnly: false };
+		let note = await client.randomNote(options);
+
+		if (!note) {
+			await client.createNote({
+				...defaultNoteProperties,
+				parentId: await selectOrCreateParentFolder(),
+				id: uuid.create(),
+				title: 'Test note',
+				body: 'Body',
+			});
+
+			note = await client.randomNote(options);
+			assert.ok(note, 'should have selected a random note');
+		}
+
+		return note;
+	};
+
+	const actions = {
+		newSubfolder: async () => {
+			const parentId = await selectOrCreateParentFolder();
+			await client.createRandomFolder(parentId, { quiet: false });
+
+			return true;
+		},
+		newToplevelFolder: async () => {
+			await client.createRandomFolder('', { quiet: false });
+			return true;
+		},
+		newNote: async () => {
+			const parentId = await selectOrCreateParentFolder();
+			await client.createRandomNote(parentId);
+
+			return true;
+		},
+		renameNote: async () => {
+			const note = await selectOrCreateWriteableNote();
+
+			await client.updateNote({
+				...note,
+				title: `Renamed (${context.randInt(0, 1000)})`,
+			});
+
+			return true;
+		},
+		updateNoteBody: async () => {
+			const note = await selectOrCreateWriteableNote();
+
+			await client.updateNote({
+				...note,
+				body: `${note.body}\n\nUpdated.\n`,
+			});
+
+			return true;
+		},
+		moveNote: async () => {
+			const note = await selectOrCreateWriteableNote();
+			const targetParent = await client.randomFolder({
+				filter: folder => folder.id !== note.parentId,
+				includeReadOnly: false,
+			});
+			if (!targetParent) return false;
+
+			await client.moveItem(note.id, targetParent.id);
+
+			return true;
+		},
+		deleteNote: async () => {
+			const target = await client.randomNote({ includeReadOnly: false });
+			if (!target) return false;
+
+			await client.deleteNote(target.id);
+			return true;
+		},
+		shareFolder: async () => {
+			const other = clientPool.randomClient(c => !c.hasSameAccount(client));
+			if (!other) return false;
+
+			const target = await client.randomFolder({
+				filter: candidate => {
+					const isToplevel = !candidate.parentId;
+					const ownedByCurrent = candidate.ownedByEmail === client.email;
+					const alreadyShared = isToplevel && candidate.isSharedWith(other.email);
+					return isToplevel && ownedByCurrent && !alreadyShared;
+				},
+				includeReadOnly: true,
+			});
+			if (!target) return false;
+
+			const readOnly = context.randInt(0, 2) === 1 && context.isJoplinCloud;
+			await client.shareFolder(target.id, other, { readOnly });
+			return true;
+		},
+		unshareFolder: async () => {
+			const target = await client.randomFolder({
+				filter: candidate => {
+					return candidate.isRootSharedItem && candidate.ownedByEmail === client.email;
+				},
+				includeReadOnly: true,
+			});
+			if (!target) return false;
+
+			const recipientIndex = context.randInt(-1, target.shareRecipients.length);
+			if (recipientIndex === -1) { // Completely remove the share
+				await client.deleteAssociatedShare(target.id);
+			} else {
+				const recipientEmail = target.shareRecipients[recipientIndex];
+				const recipient = clientPool.clientsByEmail(recipientEmail)[0];
+				assert.ok(recipient, `invalid state -- recipient ${recipientEmail} should exist`);
+				await client.removeFromShare(target.id, recipient);
+			}
+			return true;
+		},
+		deleteFolder: async () => {
+			const target = await client.randomFolder({ includeReadOnly: false });
+			if (!target) return false;
+
+			await client.deleteFolder(target.id);
+			return true;
+		},
+		moveFolderToToplevel: async () => {
+			const target = await client.randomFolder({
+				// Don't choose items that are already toplevel
+				filter: item => !!item.parentId,
+				includeReadOnly: false,
+			});
+			if (!target) return false;
+
+			await client.moveItem(target.id, '');
+			return true;
+		},
+		moveFolderTo: async () => {
+			const target = await client.randomFolder({
+				// Don't move shared folders (should not be allowed by the GUI in the main apps).
+				filter: item => !item.isRootSharedItem,
+				includeReadOnly: false,
+			});
+			if (!target) return false;
+
+			const targetDescendants = new Set(await client.allFolderDescendants(target.id));
+
+			const newParent = await client.randomFolder({
+				filter: (item) => {
+					// Avoid making the folder a child of itself
+					return !targetDescendants.has(item.id);
+				},
+				includeReadOnly: false,
+			});
+			if (!newParent) return false;
+
+			await client.moveItem(target.id, newParent.id);
+			return true;
+		},
+		newClientOnSameAccount: async () => {
+			const welcomeNoteCount = context.randInt(0, 30);
+			logger.info(`Syncing a new client on the same account ${welcomeNoteCount > 0 ? `(with ${welcomeNoteCount} initial notes)` : ''}`);
+			const createClientInitialNotes = async (client: Client) => {
+				if (welcomeNoteCount === 0) return;
+
+				// Create a new folder. Usually, new clients have a default set of
+				// welcome notes when first syncing.
+				const parentFolder = await client.createRandomFolder('', { quiet: false });
+
+				for (let i = 0; i < welcomeNoteCount; i++) {
+					await client.createRandomNote(parentFolder.id);
+				}
+			};
+
+			await client.sync();
+
+			const other = await clientPool.newWithSameAccount(client);
+			await createClientInitialNotes(other);
+
+			// Sometimes, a delay is needed between client creation
+			// and initial sync. Retry the initial sync and the checkState
+			// on failure:
+			await retryWithCount(async () => {
+				await other.sync();
+				await other.checkState();
+			}, {
+				delayOnFailure: (count) => Second * count,
+				count: 3,
+				onFail: async (error) => {
+					logger.warn('other.sync/other.checkState failed with', error, 'retrying...');
+				},
+			});
+
+			await client.sync();
+			return true;
+		},
+		removeClientsOnSameAccount: async () => {
+			const others = clientPool.othersWithSameAccount(client);
+			if (others.length === 0) return false;
+
+			for (const otherClient of others) {
+				assert.notEqual(otherClient, client);
+				await otherClient.close();
+			}
+			return true;
+		},
+		createOrUpdateMany: async () => {
+			await client.createOrUpdateMany(context.randInt(1, 512));
+			return true;
+		},
+		publishNote: async () => {
+			const note = await client.randomNote({
+				includeReadOnly: true,
+			});
+			if (!note || note.published) return false;
+
+			await client.publishNote(note.id);
+			return true;
+		},
+		unpublishNote: async () => {
+			const note = await client.randomNote({ includeReadOnly: true });
+			if (!note || !note.published) return false;
+
+			await client.unpublishNote(note.id);
+			return true;
+		},
+	};
+
+	const actionKeys = [...Object.keys(actions)] as (keyof typeof actions)[];
+
+	let result = false;
+	while (!result) { // Loop until an action was done
+		const randomAction = context.randomFrom(actionKeys);
+		logger.info(`Action: ${randomAction} in ${client.email}`);
+		result = await actions[randomAction]();
+		if (!result) {
+			logger.info(`  ${randomAction} was skipped (preconditions not met).`);
+		}
+	}
+};
+
+export default doRandomAction;

packages/tools/fuzzer/sync-fuzzer.ts

@@ -1,4 +1,3 @@
-import uuid from '@joplin/lib/uuid';
 import { join } from 'path';
 import { exists, mkdir, remove } from 'fs-extra';
 import Setting, { Env } from '@joplin/lib/models/Setting';
@@ -7,14 +6,14 @@ import Server from './Server';
 import { CleanupTask, FuzzContext } from './types';
 import ClientPool from './ClientPool';
 import retryWithCount from './utils/retryWithCount';
-import Client from './Client';
 import SeededRandom from './utils/SeededRandom';
 import { env } from 'process';
 import yargs = require('yargs');
-import { strict as assert } from 'assert';
 import openDebugSession from './utils/openDebugSession';
 import { Second } from '@joplin/utils/time';
 import { packagesDir } from './constants';
+import doRandomAction from './doRandomAction';
+import randomString from './utils/randomString';
 const { shimInit } = require('@joplin/lib/shim-init-node');
 
 const globalLogger = new Logger();
@@ -41,300 +40,53 @@ const createProfilesDirectory = async () => {
 	};
 };
 
-const doRandomAction = async (context: FuzzContext, client: Client, clientPool: ClientPool) => {
-	const selectOrCreateParentFolder = async () => {
-		let parentId = (await client.randomFolder({ includeReadOnly: false }))?.id;
-
-		// Create a toplevel folder to serve as this
-		// folder's parent if none exist yet
-		if (!parentId) {
-			parentId = uuid.create();
-			await client.createFolder({
-				parentId: '',
-				id: parentId,
-				title: 'Parent folder',
-			});
-		}
-
-		return parentId;
-	};
-
-	const defaultNoteProperties = {
-		published: false,
-	};
-
-	const selectOrCreateWriteableNote = async () => {
-		const options = { includeReadOnly: false };
-		let note = await client.randomNote(options);
-
-		if (!note) {
-			await client.createNote({
-				...defaultNoteProperties,
-				parentId: await selectOrCreateParentFolder(),
-				id: uuid.create(),
-				title: 'Test note',
-				body: 'Body',
-			});
-
-			note = await client.randomNote(options);
-			assert.ok(note, 'should have selected a random note');
-		}
-
-		return note;
-	};
-
-	const actions = {
-		newSubfolder: async () => {
-			const folderId = uuid.create();
-			const parentId = await selectOrCreateParentFolder();
-
-			await client.createFolder({
-				parentId: parentId,
-				id: folderId,
-				title: 'Subfolder',
-			});
-
-			return true;
-		},
-		newToplevelFolder: async () => {
-			const folderId = uuid.create();
-			await client.createFolder({
-				parentId: null,
-				id: folderId,
-				title: `Folder ${context.randInt(0, 1000)}`,
-			});
-
-			return true;
-		},
-		newNote: async () => {
-			const parentId = await selectOrCreateParentFolder();
-			await client.createNote({
-				...defaultNoteProperties,
-				parentId: parentId,
-				title: `Test (x${context.randInt(0, 1000)})`,
-				body: 'Testing...',
-				id: uuid.create(),
-			});
-
-			return true;
-		},
-		renameNote: async () => {
-			const note = await selectOrCreateWriteableNote();
-
-			await client.updateNote({
-				...note,
-				title: `Renamed (${context.randInt(0, 1000)})`,
-			});
-
-			return true;
-		},
-		updateNoteBody: async () => {
-			const note = await selectOrCreateWriteableNote();
-
-			await client.updateNote({
-				...note,
-				body: `${note.body}\n\nUpdated.\n`,
-			});
-
-			return true;
-		},
-		moveNote: async () => {
-			const note = await selectOrCreateWriteableNote();
-			const targetParent = await client.randomFolder({
-				filter: folder => folder.id !== note.parentId,
-				includeReadOnly: false,
-			});
-			if (!targetParent) return false;
-
-			await client.moveItem(note.id, targetParent.id);
-
-			return true;
-		},
-		deleteNote: async () => {
-			const target = await client.randomNote({ includeReadOnly: false });
-			if (!target) return false;
-
-			await client.deleteNote(target.id);
-			return true;
-		},
-		shareFolder: async () => {
-			const other = clientPool.randomClient(c => !c.hasSameAccount(client));
-			if (!other) return false;
-
-			const target = await client.randomFolder({
-				filter: candidate => {
-					const isToplevel = !candidate.parentId;
-					const ownedByCurrent = candidate.ownedByEmail === client.email;
-					const alreadyShared = isToplevel && candidate.isSharedWith(other.email);
-					return isToplevel && ownedByCurrent && !alreadyShared;
-				},
-				includeReadOnly: true,
-			});
-			if (!target) return false;
-
-			const readOnly = context.randInt(0, 2) === 1 && context.isJoplinCloud;
-			await client.shareFolder(target.id, other, { readOnly });
-			return true;
-		},
-		unshareFolder: async () => {
-			const target = await client.randomFolder({
-				filter: candidate => {
-					return candidate.isRootSharedItem && candidate.ownedByEmail === client.email;
-				},
-				includeReadOnly: true,
-			});
-			if (!target) return false;
-
-			const recipientIndex = context.randInt(-1, target.shareRecipients.length);
-			if (recipientIndex === -1) { // Completely remove the share
-				await client.deleteAssociatedShare(target.id);
-			} else {
-				const recipientEmail = target.shareRecipients[recipientIndex];
-				const recipient = clientPool.clientsByEmail(recipientEmail)[0];
-				assert.ok(recipient, `invalid state -- recipient ${recipientEmail} should exist`);
-				await client.removeFromShare(target.id, recipient);
-			}
-			return true;
-		},
-		deleteFolder: async () => {
-			const target = await client.randomFolder({ includeReadOnly: false });
-			if (!target) return false;
-
-			await client.deleteFolder(target.id);
-			return true;
-		},
-		moveFolderToToplevel: async () => {
-			const target = await client.randomFolder({
-				// Don't choose items that are already toplevel
-				filter: item => !!item.parentId,
-				includeReadOnly: false,
-			});
-			if (!target) return false;
-
-			await client.moveItem(target.id, '');
-			return true;
-		},
-		moveFolderTo: async () => {
-			const target = await client.randomFolder({
-				// Don't move shared folders (should not be allowed by the GUI in the main apps).
-				filter: item => !item.isRootSharedItem,
-				includeReadOnly: false,
-			});
-			if (!target) return false;
-
-			const targetDescendants = new Set(await client.allFolderDescendants(target.id));
-
-			const newParent = await client.randomFolder({
-				filter: (item) => {
-					// Avoid making the folder a child of itself
-					return !targetDescendants.has(item.id);
-				},
-				includeReadOnly: false,
-			});
-			if (!newParent) return false;
-
-			await client.moveItem(target.id, newParent.id);
-			return true;
-		},
-		newClientOnSameAccount: async () => {
-			const welcomeNoteCount = context.randInt(0, 30);
-			logger.info(`Syncing a new client on the same account ${welcomeNoteCount > 0 ? `(with ${welcomeNoteCount} initial notes)` : ''}`);
-			const createClientInitialNotes = async (client: Client) => {
-				if (welcomeNoteCount === 0) return;
-
-				// Create a new folder. Usually, new clients have a default set of
-				// welcome notes when first syncing.
-				const testNotesFolderId = uuid.create();
-				await client.createFolder({
-					id: testNotesFolderId,
-					title: 'Test -- from secondary client',
-					parentId: '',
-				});
-
-				for (let i = 0; i < welcomeNoteCount; i++) {
-					await client.createNote({
-						...defaultNoteProperties,
-						parentId: testNotesFolderId,
-						id: uuid.create(),
-						title: `Test note ${i}/${welcomeNoteCount}`,
-						body: `Test note (in account ${client.email}), created ${Date.now()}.`,
-					});
-				}
-			};
-
-			await client.sync();
-
-			const other = await clientPool.newWithSameAccount(client);
-			await createClientInitialNotes(other);
-
-			// Sometimes, a delay is needed between client creation
-			// and initial sync. Retry the initial sync and the checkState
-			// on failure:
-			await retryWithCount(async () => {
-				await other.sync();
-				await other.checkState();
-			}, {
-				delayOnFailure: (count) => Second * count,
-				count: 3,
-				onFail: async (error) => {
-					logger.warn('other.sync/other.checkState failed with', error, 'retrying...');
-				},
-			});
-
-			await client.sync();
-			return true;
-		},
-		removeClientsOnSameAccount: async () => {
-			const others = clientPool.othersWithSameAccount(client);
-			if (others.length === 0) return false;
-
-			for (const otherClient of others) {
-				assert.notEqual(otherClient, client);
-				await otherClient.close();
-			}
-			return true;
-		},
-		publishNote: async () => {
-			const note = await client.randomNote({
-				includeReadOnly: true,
-			});
-			if (!note || note.published) return false;
-
-			await client.publishNote(note.id);
-			return true;
-		},
-		unpublishNote: async () => {
-			const note = await client.randomNote({ includeReadOnly: true });
-			if (!note || !note.published) return false;
-
-			await client.unpublishNote(note.id);
-			return true;
-		},
-	};
-
-	const actionKeys = [...Object.keys(actions)] as (keyof typeof actions)[];
-
-	let result = false;
-	while (!result) { // Loop until an action was done
-		const randomAction = actionKeys[context.randInt(0, actionKeys.length)];
-		logger.info(`Action: ${randomAction} in ${client.email}`);
-		result = await actions[randomAction]();
-		if (!result) {
-			logger.info(`  ${randomAction} was skipped (preconditions not met).`);
-		}
-	}
-};
-
 interface Options {
 	seed: number;
 	maximumSteps: number;
 	maximumStepsBetweenSyncs: number;
+	enableE2ee: boolean;
+	randomStrings: boolean;
 	clientCount: number;
+	keepAccountsOnClose: boolean;
 
 	serverPath: string;
 	isJoplinCloud: boolean;
 }
 
+const createContext = (options: Options, server: Server, profilesDirectory: string) => {
+	const random = new SeededRandom(options.seed);
+	// Use a separate random number generator for strings. This prevents
+	// the random strings setting from affecting the other output.
+	const stringRandom = new SeededRandom(random.next());
+
+	if (options.isJoplinCloud) {
+		logger.info('Sync target: Joplin Cloud');
+	}
+
+	let stringCount = 0;
+	const randomStringGenerator = (() => {
+		if (options.randomStrings) {
+			return randomString((min, max) => stringRandom.nextInRange(min, max));
+		} else {
+			return (_targetLength: number) => `Placeholder (x${stringCount++})`;
+		}
+	})();
+
+	const fuzzContext: FuzzContext = {
+		serverUrl: server.url,
+		isJoplinCloud: options.isJoplinCloud,
+		enableE2ee: options.enableE2ee,
+		baseDir: profilesDirectory,
+
+		execApi: server.execApi.bind(server),
+		randInt: (a, b) => random.nextInRange(a, b),
+		randomFrom: (data) => data[random.nextInRange(0, data.length)],
+		randomString: randomStringGenerator,
+		keepAccounts: options.keepAccountsOnClose,
+	};
+	return fuzzContext;
+};
+
 const main = async (options: Options) => {
 	shimInit();
 	Setting.setConstant('env', Env.Dev);
@@ -378,25 +130,14 @@ const main = async (options: Options) => {
 		cleanupTasks.push(profilesDirectory.remove);
 
 		logger.info('Starting with seed', options.seed);
-		const random = new SeededRandom(options.seed);
 
-		if (options.isJoplinCloud) {
+		const fuzzContext = createContext(options, server, profilesDirectory.path);
-			logger.info('Sync target: Joplin Cloud');
-		}
-
-		const fuzzContext: FuzzContext = {
-			serverUrl: joplinServerUrl,
-			isJoplinCloud: options.isJoplinCloud,
-			baseDir: profilesDirectory.path,
-			execApi: server.execApi.bind(server),
-			randInt: (a, b) => random.nextInRange(a, b),
-		};
 		clientPool = await ClientPool.create(
 			fuzzContext,
 			options.clientCount,
 			task => { cleanupTasks.push(task); },
 		);
-		await clientPool.syncAll();
+		await clientPool.createInitialItemsAndSync();
 
 		const maxSteps = options.maximumSteps;
 		for (let stepIndex = 1; maxSteps <= 0 || stepIndex <= maxSteps; stepIndex++) {
@@ -475,6 +216,21 @@ void yargs
 					default: 3,
 					defaultDescription: 'Number of client apps to create.',
 				},
+				'keep-accounts': {
+					type: 'boolean',
+					default: false,
+					defaultDescription: 'Whether to keep the created Joplin Server users after exiting. Default is to try to clean up, removing old accounts when exiting.',
+				},
+				'enable-e2ee': {
+					type: 'boolean',
+					default: true,
+					description: 'Whether to enable end-to-end encryption',
+				},
+				'random-strings': {
+					type: 'boolean',
+					default: true,
+					description: 'Whether to generate text using pseudorandom Unicode characters. Disabling this can simplify debugging.',
+				},
 				'joplin-cloud': {
 					type: 'string',
 					default: '',
@@ -494,6 +250,9 @@ void yargs
 				serverPath: serverPath,
 				isJoplinCloud: !!argv.joplinCloud,
 				maximumStepsBetweenSyncs: argv['steps-between-syncs'],
+				keepAccountsOnClose: argv.keepAccounts,
+				enableE2ee: argv.enableE2ee,
+				randomStrings: argv.randomStrings,
 			});
 		},
 	)

packages/tools/fuzzer/types.ts

@@ -46,9 +46,14 @@ export const assertIsFolder: (item: TreeItem)=> asserts item is FolderRecord = i
 export interface FuzzContext {
 	serverUrl: string;
 	isJoplinCloud: boolean;
+	keepAccounts: boolean;
+	enableE2ee: boolean;
 	baseDir: string;
+
 	execApi: (method: HttpMethod, route: string, debugAction: Json)=> Promise<Json>;
 	randInt: (low: number, high: number)=> number;
+	randomString: (targetLength: number)=> string;
+	randomFrom: <T> (data: T[])=> T;
 }
 
 export interface RandomFolderOptions {

packages/tools/fuzzer/utils/ProgressBar.ts

@@ -0,0 +1,28 @@
+import { stdout } from 'process';
+
+export default class ProgressBar {
+	private isFirst_ = true;
+	private lastLength_ = 0;
+	public constructor(private prefix_: string) {}
+	public update(countDone: number, total: number) {
+		if (!stdout.isTTY) return;
+
+		if (this.isFirst_) {
+			this.isFirst_ = false;
+		}
+
+		const percent = Math.round(countDone / total * 100);
+		const message = `\r${this.prefix_}: ${percent}% (${countDone}/${total})`;
+		stdout.write(message.padEnd(this.lastLength_));
+
+		this.lastLength_ = message.length;
+	}
+
+	public complete() {
+		if (!this.isFirst_) {
+			stdout.write(`\r${this.prefix_}: Done`.padEnd(this.lastLength_));
+			stdout.write('\n');
+		}
+	}
+}
+

packages/tools/fuzzer/utils/SeededRandom.ts

@@ -27,8 +27,8 @@ export default class SeededRandom {
 	private nextStep_: bigint = step;
 	private halfSize_ = BigInt(32);
 
-	public constructor(seed: number) {
+	public constructor(seed: number|bigint) {
-		this.value_ = BigInt(seed);
+		this.value_ = typeof seed === 'number' ? BigInt(seed) : seed;
 	}
 
 	public next() {

packages/tools/fuzzer/utils/logDiffDebug.ts

@@ -0,0 +1,43 @@
+import Logger from '@joplin/utils/Logger';
+
+const logger = Logger.create('logDiffDebug');
+
+// Provides additional debugging information for differing strings
+const logDiffDebug = (actual: string, expected: string) => {
+	const actualBinary = new Uint8Array(new TextEncoder().encode(actual));
+	const expectedBinary = new Uint8Array(new TextEncoder().encode(expected));
+	for (let i = 0; i < actualBinary.length; i++) {
+		if (i >= expectedBinary.length) {
+			logger.warn('Actual is longer than expected');
+			break;
+		}
+
+		if (expectedBinary[i] !== actualBinary[i]) {
+			logger.warn(
+				'First binary difference at position', i, `(0x${i.toString(16)})`, ': ', expectedBinary[i], '!=', actualBinary[i], '(expected != actual)',
+				'\n\tContext: expected[i-3:i+5] = ', [...expectedBinary.slice(i - 3, i + 5)],
+				'\n\tContext: actual[i-3 : i+5] = ', [...actualBinary.slice(i - 3, i + 5)],
+				'\n\tactual.byteLength = ', actualBinary.length, ', expected.byteLength = ', expectedBinary.length,
+			);
+			break;
+		}
+	}
+
+	// Also log information about the last difference
+	for (let i = 0; i < Math.min(actualBinary.length, expectedBinary.length); i++) {
+		const indexExpected = expectedBinary.length - i - 1;
+		const indexActual = actualBinary.length - i - 1;
+		if (expectedBinary[indexExpected] !== actualBinary[indexActual]) {
+			logger.warn(
+				'Last binary difference (working from end)', ': ',
+				expectedBinary[indexExpected], '!=', actualBinary[indexActual], `(expected[${indexExpected}] != actual[${indexActual}])`,
+				'\n\tContext: expected[a-6:a+3] = ', [...expectedBinary.slice(indexExpected - 6, indexExpected + 3)],
+				'\n\tContext: actual[b-6 : b+3] = ', [...actualBinary.slice(indexActual - 6, indexActual + 3)],
+				'\n\twhere expected.byteLength = ', expectedBinary.byteLength, 'and actual.byteLength = ', actualBinary.byteLength,
+			);
+			break;
+		}
+	}
+};
+
+export default logDiffDebug;

packages/tools/fuzzer/utils/randomString.ts

@@ -0,0 +1,29 @@
+type OnRandomInt = (low: number, high: number)=> number;
+
+const randomString = (nextRandomInteger: OnRandomInt) => (length: number) => {
+	const charCodes = [];
+	for (let i = 0; i < length; i++) {
+		charCodes.push(nextRandomInteger(0, 0xFFFF));
+	}
+
+	let text = String.fromCharCode(...charCodes);
+	// Normalize to avoid differences when reading/writing content from Joplin clients.
+	// TODO: Can the note comparison logic be adjusted to remove some of this?
+	text = text.normalize();
+	text = text.replace(/[\r\b\f\v\0\u007F]/g, '!');
+	text = text.replace(/\p{C}/ug, '-'); // Other control characters
+
+	// Remove invalid UTF-8
+	text = new TextDecoder().decode(new TextEncoder().encode(text));
+
+	// Attempt to work around issues related to unexpected differences in items when reading from
+	// the Joplin client: Remove certain invalid unicode:
+	if ('toWellFormed' in String.prototype && typeof String.prototype.toWellFormed === 'function') {
+		// toWellFormed requires Node >= v20
+		text = String.prototype.toWellFormed.call(text);
+	}
+
+	return text;
+};
+
+export default randomString;