Chore: Resolves #12283: Server: Add fuzzer for detecting sync bugs (#12592)

Co-authored-by: Laurent Cozic <laurent22@users.noreply.github.com>

.eslintignore

@@ -937,7 +937,6 @@ packages/default-plugins/commands/editPatch.js
 packages/default-plugins/utils/getCurrentCommitHash.js
 packages/default-plugins/utils/getPathToPatchFileFor.js
 packages/default-plugins/utils/readRepositoryJson.js
-packages/default-plugins/utils/waitForCliInput.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5BuiltInOptions.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.test.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.js
@@ -1649,6 +1648,18 @@ packages/tools/checkIgnoredFiles.js
 packages/tools/checkLibPaths.test.js
 packages/tools/checkLibPaths.js
 packages/tools/convertThemesToCss.js
+packages/tools/fuzzer/ActionTracker.js
+packages/tools/fuzzer/Client.js
+packages/tools/fuzzer/ClientPool.js
+packages/tools/fuzzer/Server.js
+packages/tools/fuzzer/constants.js
+packages/tools/fuzzer/sync-fuzzer.js
+packages/tools/fuzzer/types.js
+packages/tools/fuzzer/utils/SeededRandom.js
+packages/tools/fuzzer/utils/getNumberProperty.js
+packages/tools/fuzzer/utils/getProperty.js
+packages/tools/fuzzer/utils/getStringProperty.js
+packages/tools/fuzzer/utils/retryWithCount.js
 packages/tools/generate-database-types.js
 packages/tools/generate-images.js
 packages/tools/git-changelog.test.js

.eslintrc.js

@@ -23,6 +23,7 @@ module.exports = {
 		'FileSystemCreateWritableOptions': 'readonly',
 		'FileSystemHandle': 'readonly',
 		'IDBTransactionMode': 'readonly',
+		'BigInt': 'readonly',
 		'globalThis': 'readonly',
 
 		// ServiceWorker

.gitignore

@@ -912,7 +912,6 @@ packages/default-plugins/commands/editPatch.js
 packages/default-plugins/utils/getCurrentCommitHash.js
 packages/default-plugins/utils/getPathToPatchFileFor.js
 packages/default-plugins/utils/readRepositoryJson.js
-packages/default-plugins/utils/waitForCliInput.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5BuiltInOptions.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.test.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.js
@@ -1624,6 +1623,18 @@ packages/tools/checkIgnoredFiles.js
 packages/tools/checkLibPaths.test.js
 packages/tools/checkLibPaths.js
 packages/tools/convertThemesToCss.js
+packages/tools/fuzzer/ActionTracker.js
+packages/tools/fuzzer/Client.js
+packages/tools/fuzzer/ClientPool.js
+packages/tools/fuzzer/Server.js
+packages/tools/fuzzer/constants.js
+packages/tools/fuzzer/sync-fuzzer.js
+packages/tools/fuzzer/types.js
+packages/tools/fuzzer/utils/SeededRandom.js
+packages/tools/fuzzer/utils/getNumberProperty.js
+packages/tools/fuzzer/utils/getProperty.js
+packages/tools/fuzzer/utils/getStringProperty.js
+packages/tools/fuzzer/utils/retryWithCount.js
 packages/tools/generate-database-types.js
 packages/tools/generate-images.js
 packages/tools/git-changelog.test.js

package.json

@@ -38,6 +38,7 @@
     "linter-precommit": "eslint --resolve-plugins-relative-to . --fix --ext .js --ext .jsx --ext .ts --ext .tsx",
     "linter": "eslint --resolve-plugins-relative-to . --fix --quiet --ext .js --ext .jsx --ext .ts --ext .tsx",
     "packageJsonLint": "node ./packages/tools/packageJsonLint.js",
+    "syncFuzzer": "node ./packages/tools/fuzzer/sync-fuzzer.js",
     "postinstall": "husky && gulp build",
     "postPreReleasesToForum": "node ./packages/tools/postPreReleasesToForum",
     "publishAll": "git pull && yarn buildParallel && lerna version --yes --no-private --no-git-tag-version && gulp completePublishAll",

packages/app-cli/app/app.ts

@@ -419,6 +419,11 @@ class Application extends BaseApplication {
 
 		this.initRedux();
 
+		// Since the settings need to be loaded before the store is created, it will never
+		// receive the SETTING_UPDATE_ALL even, which mean state.settings will not be
+		// initialised. So we manually call dispatchUpdateAll() to force an update.
+		Setting.dispatchUpdateAll();
+
 		if (!shim.sharpEnabled()) this.logger().warn('Sharp is disabled - certain image-related features will not be available');
 
 		initializeCommandService(this.store(), Setting.value('env') === Env.Dev);
@@ -461,11 +466,6 @@ class Application extends BaseApplication {
 			this.gui_.setLogger(this.logger());
 			await this.gui_.start();
 
-			// Since the settings need to be loaded before the store is created, it will never
-			// receive the SETTING_UPDATE_ALL even, which mean state.settings will not be
-			// initialised. So we manually call dispatchUpdateAll() to force an update.
-			Setting.dispatchUpdateAll();
-
 			// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
 			await refreshFolders((action: any) => this.store().dispatch(action), '');
 

packages/app-cli/app/command-e2ee.ts

@@ -26,6 +26,7 @@ class Command extends BaseCommand {
 			['-v, --verbose', 'More verbose output for the `target-status` command'],
 			['-o, --output <directory>', 'Output directory'],
 			['--retry-failed-items', 'Applies to `decrypt` command - retries decrypting items that previously could not be decrypted.'],
+			['-f, --force', 'Do not ask for input on failure'],
 		];
 	}
 
@@ -67,7 +68,7 @@ class Command extends BaseCommand {
 					this.stdout(line.join('\n'));
 					break;
 				} catch (error) {
-					if (error.code === 'masterKeyNotLoaded') {
+					if (error.code === 'masterKeyNotLoaded' && !args.options.force) {
 						const ok = await askForMasterKey(error);
 						if (!ok) return;
 						continue;

packages/default-plugins/buildDefaultPlugins.ts

@@ -8,9 +8,9 @@ import { chdir, cwd } from 'process';
 import { execCommand } from '@joplin/utils';
 import { glob } from 'glob';
 import readRepositoryJson from './utils/readRepositoryJson';
-import waitForCliInput from './utils/waitForCliInput';
 import getPathToPatchFileFor from './utils/getPathToPatchFileFor';
 import getCurrentCommitHash from './utils/getCurrentCommitHash';
+import { waitForCliInput } from '@joplin/utils/cli';
 
 interface Options {
 	beforeInstall: (buildDir: string, pluginName: string)=> Promise<void>;

packages/default-plugins/commands/editPatch.ts

@@ -1,7 +1,7 @@
 import { execCommand } from '@joplin/utils';
-import waitForCliInput from '../utils/waitForCliInput';
 import { copy } from 'fs-extra';
 import { join } from 'path';
+import { waitForCliInput } from '@joplin/utils/cli';
 import buildDefaultPlugins from '../buildDefaultPlugins';
 import getPathToPatchFileFor from '../utils/getPathToPatchFileFor';
 

packages/lib/ClipperServer.ts

@@ -98,7 +98,7 @@ export default class ClipperServer {
 		});
 	}
 
-	public async findAvailablePort() {
+	public async findAvailablePort(): Promise<number> {
 		const tcpPortUsed = require('tcp-port-used');
 
 		let state = null;

packages/server/src/models/ItemModel.ts

@@ -914,8 +914,14 @@ export default class ItemModel extends BaseModel<Item> {
 			const share = await this.models().share().byItemId(item.id);
 			if (!share) throw new Error(`Cannot find share associated with item ${item.id}`);
 			const userShare = await this.models().shareUser().byShareAndUserId(share.id, userId);
-			if (!userShare) return;
-			await this.models().shareUser().delete(userShare.id);
+
+			if (userShare) {
+				// Leave the share
+				await this.models().shareUser().delete(userShare.id);
+			} else if (share.owner_id === userId) {
+				// Delete the share
+				await this.models().share().delete(share.id);
+			}
 		} else {
 			await this.delete(item.id);
 		}

packages/tools/.gitignore

@@ -4,3 +4,4 @@ patreon_oauth_token.txt
 *.po~
 *.mo
 *.mo~
+fuzzer/profiles-tmp/

packages/tools/cspell/dictionary4.txt

@@ -183,6 +183,7 @@ topagency
 esbuild
 mapbox
 outfile
+fuzzer
 Freespinny
 BestEtf
 Etf

packages/tools/fuzzer/ActionTracker.ts

@@ -0,0 +1,387 @@
+import { strict as assert } from 'assert';
+import { ActionableClient, FolderData, FolderMetadata, FuzzContext, ItemId, NoteData, TreeItem, isFolder } from './types';
+import type Client from './Client';
+
+interface ClientData {
+	childIds: ItemId[];
+	// Shared folders belonging to the client
+	sharedFolderIds: ItemId[];
+}
+
+class ActionTracker {
+	private idToItem_: Map<ItemId, TreeItem> = new Map();
+	private tree_: Map<string, ClientData> = new Map();
+	public constructor(private readonly context_: FuzzContext) {}
+
+	private checkRep_() {
+		const checkItem = (itemId: ItemId) => {
+			assert.match(itemId, /^[a-zA-Z0-9]{32}$/, 'item IDs should be 32 character alphanumeric strings');
+
+			const item = this.idToItem_.get(itemId);
+			assert.ok(!!item, `should find item with ID ${itemId}`);
+
+			if (item.parentId) {
+				const parent = this.idToItem_.get(item.parentId);
+				assert.ok(parent, `should find parent (id: ${item.parentId})`);
+
+				assert.ok(isFolder(parent), 'parent should be a folder');
+				assert.ok(parent.childIds.includes(itemId), 'parent should include the current item in its children');
+			}
+
+			if (isFolder(item)) {
+				for (const childId of item.childIds) {
+					checkItem(childId);
+				}
+
+				assert.equal(
+					item.childIds.length,
+					[...new Set(item.childIds)].length,
+					'child IDs should be unique',
+				);
+			}
+		};
+
+		for (const clientData of this.tree_.values()) {
+			for (const childId of clientData.childIds) {
+				assert.ok(this.idToItem_.has(childId), `root item ${childId} should exist`);
+
+				const item = this.idToItem_.get(childId);
+				assert.ok(!!item);
+				assert.equal(item.parentId, '', `${childId} should not have a parent`);
+
+				checkItem(childId);
+			}
+		}
+	}
+
+	public track(client: { email: string }) {
+		const clientId = client.email;
+		this.tree_.set(clientId, {
+			childIds: [],
+			sharedFolderIds: [],
+		});
+
+		const getChildIds = (itemId: ItemId) => {
+			const item = this.idToItem_.get(itemId);
+			if (!item || !isFolder(item)) return [];
+			return item.childIds;
+		};
+		const updateChildren = (parentId: ItemId, updateFn: (oldChildren: ItemId[])=> ItemId[]) => {
+			const parent = this.idToItem_.get(parentId);
+			if (!parent) throw new Error(`Parent with ID ${parentId} not found.`);
+			if (!isFolder(parent)) throw new Error(`Item ${parentId} is not a folder`);
+
+			this.idToItem_.set(parentId, {
+				...parent,
+				childIds: updateFn(parent.childIds),
+			});
+		};
+		const addRootItem = (itemId: ItemId) => {
+			const clientData = this.tree_.get(clientId);
+			if (!clientData.childIds.includes(itemId)) {
+				this.tree_.set(clientId, {
+					...clientData,
+					childIds: [...clientData.childIds, itemId],
+				});
+			}
+		};
+
+		// Returns true iff the given item ID is now unused.
+		const removeRootItem = (itemId: ItemId) => {
+			const removeForClient = (clientId: string) => {
+				const clientData = this.tree_.get(clientId);
+				const childIds = clientData.childIds;
+
+				if (childIds.includes(itemId)) {
+					const newChildIds = childIds.filter(otherId => otherId !== itemId);
+					this.tree_.set(clientId, {
+						...clientData,
+						childIds: newChildIds,
+					});
+					return true;
+				}
+
+				return false;
+			};
+
+			const hasBeenCompletelyRemoved = () => {
+				for (const clientData of this.tree_.values()) {
+					if (clientData.childIds.includes(itemId)) {
+						return false;
+					}
+				}
+				return true;
+			};
+
+			const isOwnedByThis = this.tree_.get(clientId).sharedFolderIds.includes(itemId);
+
+			if (isOwnedByThis) { // Unshare
+				let removed = false;
+				for (const id of this.tree_.keys()) {
+					const result = removeForClient(id);
+					removed ||= result;
+				}
+
+				const clientData = this.tree_.get(clientId);
+				this.tree_.set(clientId, {
+					...clientData,
+					sharedFolderIds: clientData.sharedFolderIds.filter(id => id !== itemId),
+				});
+
+				// At this point, the item shouldn't be a child of any clients:
+				assert.ok(hasBeenCompletelyRemoved(), 'item should be removed from all clients');
+				assert.ok(removed, 'should be a toplevel item');
+
+				// The item is unshared and can be removed entirely
+				return true;
+			} else {
+				// Otherwise, even if part of a share, removing the
+				// notebook just leaves the share.
+				const removed = removeForClient(clientId);
+				assert.ok(removed, 'should be a toplevel item');
+
+				if (hasBeenCompletelyRemoved()) {
+					return true;
+				}
+			}
+
+			return false;
+		};
+		const addChild = (parentId: ItemId, childId: ItemId) => {
+			if (parentId) {
+				updateChildren(parentId, (oldChildren) => {
+					if (oldChildren.includes(childId)) return oldChildren;
+					return [...oldChildren, childId];
+				});
+			} else {
+				addRootItem(childId);
+			}
+		};
+		const removeChild = (parentId: ItemId, childId: ItemId) => {
+			if (!parentId) {
+				removeRootItem(childId);
+			} else {
+				updateChildren(parentId, (oldChildren) => {
+					return oldChildren.filter(otherId => otherId !== childId);
+				});
+			}
+		};
+		const removeItemRecursive = (id: ItemId) => {
+			const item = this.idToItem_.get(id);
+			if (!item) throw new Error(`Item with ID ${id} not found.`);
+
+			if (item.parentId) {
+				// The parent may already be removed
+				if (this.idToItem_.has(item.parentId)) {
+					removeChild(item.parentId, item.id);
+				}
+
+				this.idToItem_.delete(id);
+			} else {
+				const idIsUnused = removeRootItem(item.id);
+				if (idIsUnused) {
+					this.idToItem_.delete(id);
+				}
+			}
+
+			if (isFolder(item)) {
+				for (const childId of item.childIds) {
+					const child = this.idToItem_.get(childId);
+					assert.equal(child?.parentId, id, `child ${childId} should have accurate parent ID`);
+
+					removeItemRecursive(childId);
+				}
+			}
+		};
+		const mapItems = <T> (map: (item: TreeItem)=> T) => {
+			const workList: ItemId[] = [...this.tree_.get(clientId).childIds];
+			const result: T[] = [];
+
+			while (workList.length > 0) {
+				const id = workList.pop();
+				const item = this.idToItem_.get(id);
+				if (!item) throw new Error(`Not found: ${id}`);
+
+				result.push(map(item));
+
+				if (isFolder(item)) {
+					for (const childId of item.childIds) {
+						workList.push(childId);
+					}
+				}
+			}
+
+			return result;
+		};
+
+		const listFoldersDetailed = () => {
+			return mapItems((item): FolderData => {
+				return isFolder(item) ? item : null;
+			}).filter(item => !!item);
+		};
+
+		const tracker: ActionableClient = {
+			createNote: (data: NoteData) => {
+				assert.ok(!!data.parentId, `note ${data.id} should have a parentId`);
+				assert.ok(!this.idToItem_.has(data.id), `note ${data.id} should not yet exist`);
+				this.idToItem_.set(data.id, {
+					...data,
+				});
+				addChild(data.parentId, data.id);
+
+				this.checkRep_();
+				return Promise.resolve();
+			},
+			updateNote: (data: NoteData) => {
+				const oldItem = this.idToItem_.get(data.id);
+				assert.ok(oldItem, `note ${data.id} should exist`);
+				assert.ok(!!data.parentId, `note ${data.id} should have a parentId`);
+
+				removeChild(oldItem.parentId, data.id);
+				this.idToItem_.set(data.id, {
+					...data,
+				});
+				addChild(data.parentId, data.id);
+
+				this.checkRep_();
+				return Promise.resolve();
+			},
+			createFolder: (data: FolderMetadata) => {
+				this.idToItem_.set(data.id, {
+					...data,
+					parentId: data.parentId ?? '',
+					childIds: getChildIds(data.id),
+					isShareRoot: false,
+				});
+				addChild(data.parentId, data.id);
+
+				this.checkRep_();
+				return Promise.resolve();
+			},
+			deleteFolder: (id: ItemId) => {
+				this.checkRep_();
+
+				const item = this.idToItem_.get(id);
+				if (!item) throw new Error(`Not found ${id}`);
+				if (!isFolder(item)) throw new Error(`Not a folder ${id}`);
+
+				removeItemRecursive(id);
+
+				this.checkRep_();
+				return Promise.resolve();
+			},
+			shareFolder: (id: ItemId, shareWith: Client) => {
+				const shareWithChildIds = this.tree_.get(shareWith.email).childIds;
+				if (shareWithChildIds.includes(id)) {
+					throw new Error(`Folder ${id} already shared with ${shareWith.email}`);
+				}
+				assert.ok(this.idToItem_.has(id), 'should exist');
+
+				const sharerClient = this.tree_.get(clientId);
+				if (!sharerClient.sharedFolderIds.includes(id)) {
+					this.tree_.set(clientId, {
+						...sharerClient,
+						sharedFolderIds: [...sharerClient.sharedFolderIds, id],
+					});
+				}
+
+				this.tree_.set(shareWith.email, {
+					...this.tree_.get(shareWith.email),
+					childIds: [...shareWithChildIds, id],
+				});
+
+				this.idToItem_.set(id, {
+					...this.idToItem_.get(id),
+					isShareRoot: true,
+				});
+
+				this.checkRep_();
+				return Promise.resolve();
+			},
+			moveItem: (itemId, newParentId) => {
+				const item = this.idToItem_.get(itemId);
+				assert.ok(item, `item with ${itemId} should exist`);
+
+				if (newParentId) {
+					const parent = this.idToItem_.get(newParentId);
+					assert.ok(parent, `parent with ID ${newParentId} should exist`);
+				} else {
+					assert.equal(newParentId, '', 'parentId should be empty if a toplevel folder');
+				}
+
+				if (isFolder(item)) {
+					assert.equal(item.isShareRoot, false, 'cannot move toplevel shared folders without first unsharing');
+				}
+
+				removeChild(item.parentId, itemId);
+				addChild(newParentId, itemId);
+				this.idToItem_.set(itemId, {
+					...item,
+					parentId: newParentId,
+				});
+
+				this.checkRep_();
+				return Promise.resolve();
+			},
+			sync: () => Promise.resolve(),
+			listNotes: () => {
+				const notes = mapItems(item => {
+					return isFolder(item) ? null : item;
+				}).filter(item => !!item);
+
+				this.checkRep_();
+				return Promise.resolve(notes);
+			},
+			listFolders: () => {
+				this.checkRep_();
+				const folderData = listFoldersDetailed().map(item => ({
+					id: item.id,
+					title: item.title,
+					parentId: item.parentId,
+				}));
+
+				return Promise.resolve(folderData);
+			},
+			allFolderDescendants: (parentId) => {
+				this.checkRep_();
+
+				const descendants: ItemId[] = [];
+				const addDescendants = (id: ItemId) => {
+					const item = this.idToItem_.get(id);
+					assert.ok(isFolder(item), 'should be a folder');
+
+					for (const id of item.childIds) {
+						descendants.push(id);
+
+						const item = this.idToItem_.get(id);
+						if (isFolder(item)) {
+							addDescendants(item.id);
+						}
+					}
+				};
+
+				descendants.push(parentId);
+				addDescendants(parentId);
+
+				return Promise.resolve(descendants);
+			},
+			randomFolder: async (options) => {
+				let folders = listFoldersDetailed();
+				if (options.filter) {
+					folders = folders.filter(options.filter);
+				}
+
+				const folderIndex = this.context_.randInt(0, folders.length);
+				return folders.length ? folders[folderIndex] : null;
+			},
+			randomNote: async () => {
+				const notes = await tracker.listNotes();
+				const noteIndex = this.context_.randInt(0, notes.length);
+				return notes.length ? notes[noteIndex] : null;
+			},
+		};
+		return tracker;
+	}
+}
+
+export default ActionTracker;

packages/tools/fuzzer/Client.ts

@@ -0,0 +1,420 @@
+import uuid, { createSecureRandom } from '@joplin/lib/uuid';
+import { ActionableClient, FolderMetadata, FuzzContext, HttpMethod, ItemId, Json, NoteData, RandomFolderOptions, UserData } from './types';
+import { join } from 'path';
+import { mkdir } from 'fs-extra';
+import getStringProperty from './utils/getStringProperty';
+import { strict as assert } from 'assert';
+import ClipperServer from '@joplin/lib/ClipperServer';
+import ActionTracker from './ActionTracker';
+import Logger from '@joplin/utils/Logger';
+import execa = require('execa');
+import { cliDirectory } from './constants';
+import { commandToString } from '@joplin/utils';
+import { quotePath } from '@joplin/utils/path';
+import getNumberProperty from './utils/getNumberProperty';
+import retryWithCount from './utils/retryWithCount';
+
+const logger = Logger.create('Client');
+
+
+class Client implements ActionableClient {
+	public readonly email: string;
+
+	public static async create(actionTracker: ActionTracker, context: FuzzContext) {
+		const id = uuid.create();
+		const profileDirectory = join(context.baseDir, id);
+		await mkdir(profileDirectory);
+
+		const email = `${id}@localhost`;
+		const password = createSecureRandom();
+		const apiOutput = await context.execApi('POST', 'api/users', {
+			email,
+		});
+		const serverId = getStringProperty(apiOutput, 'id');
+
+		// The password needs to be set *after* creating the user.
+		const userRoute = `api/users/${encodeURIComponent(serverId)}`;
+		await context.execApi('PATCH', userRoute, {
+			email,
+			password,
+			email_confirmed: 1,
+		});
+
+		const closeAccount = async () => {
+			await context.execApi('DELETE', userRoute, {});
+		};
+
+		try {
+			const userData = {
+				email: getStringProperty(apiOutput, 'email'),
+				password,
+			};
+
+			assert.equal(email, userData.email);
+
+			const apiToken = createSecureRandom().replace(/[-]/g, '_');
+			const apiPort = await ClipperServer.instance().findAvailablePort();
+
+			const client = new Client(
+				actionTracker.track({ email }),
+				userData,
+				profileDirectory,
+				apiPort,
+				apiToken,
+				closeAccount,
+			);
+
+			// Joplin Server sync
+			await client.execCliCommand_('config', 'sync.target', '9');
+			await client.execCliCommand_('config', 'sync.9.path', context.serverUrl);
+			await client.execCliCommand_('config', 'sync.9.username', userData.email);
+			await client.execCliCommand_('config', 'sync.9.password', userData.password);
+			await client.execCliCommand_('config', 'api.token', apiToken);
+			await client.execCliCommand_('config', 'api.port', String(apiPort));
+
+			const e2eePassword = createSecureRandom().replace(/^-/, '_');
+			await client.execCliCommand_('e2ee', 'enable', '--password', e2eePassword);
+			logger.info('Created and configured client');
+
+			// Run asynchronously -- the API server command doesn't exit until the server
+			// is closed.
+			void (async () => {
+				try {
+					await client.execCliCommand_('server', 'start');
+				} catch (error) {
+					logger.info('API server exited');
+					logger.debug('API server exit status', error);
+				}
+			})();
+
+			await client.sync();
+			return client;
+		} catch (error) {
+			await closeAccount();
+			throw error;
+		}
+	}
+
+	private constructor(
+		private readonly tracker_: ActionableClient,
+		userData: UserData,
+		private readonly profileDirectory: string,
+		private readonly apiPort_: number,
+		private readonly apiToken_: string,
+		private readonly cleanUp_: ()=> Promise<void>,
+	) {
+		this.email = userData.email;
+	}
+
+	public async close() {
+		await this.execCliCommand_('server', 'stop');
+		await this.cleanUp_();
+	}
+
+	private get cliCommandArguments() {
+		return [
+			'start-no-build',
+			'--profile', this.profileDirectory,
+			'--env', 'dev',
+		];
+	}
+
+	public getHelpText() {
+		return [
+			`Client ${this.email}:`,
+			`\tCommand: cd ${quotePath(cliDirectory)} && ${commandToString('yarn', this.cliCommandArguments)}`,
+		].join('\n');
+	}
+
+	private async execCliCommand_(commandName: string, ...args: string[]) {
+		assert.match(commandName, /^[a-z]/, 'Command name must start with a lowercase letter.');
+		const commandResult = await execa('yarn', [
+			...this.cliCommandArguments,
+			commandName,
+			...args,
+		], {
+			cwd: cliDirectory,
+			// Connects /dev/null to stdin
+			stdin: 'ignore',
+		});
+		logger.debug('Ran command: ', commandResult.command, commandResult.exitCode);
+		logger.debug('     Output: ', commandResult.stdout);
+		return commandResult;
+	}
+
+	// eslint-disable-next-line no-dupe-class-members -- This is not a duplicate class member
+	private async execApiCommand_(method: 'GET', route: string): Promise<Json>;
+	// eslint-disable-next-line no-dupe-class-members -- This is not a duplicate class member
+	private async execApiCommand_(method: 'POST'|'PUT', route: string, data: Json): Promise<Json>;
+	// eslint-disable-next-line no-dupe-class-members -- This is not a duplicate class member
+	private async execApiCommand_(method: HttpMethod, route: string, data: Json|null = null): Promise<Json> {
+		route = route.replace(/^[/]/, '');
+		const url = new URL(`http://localhost:${this.apiPort_}/${route}`);
+		url.searchParams.append('token', this.apiToken_);
+
+		const response = await fetch(url, {
+			method,
+			body: data ? JSON.stringify(data) : undefined,
+		});
+
+		if (!response.ok) {
+			throw new Error(`Request to ${route} failed with error: ${await response.text()}`);
+		}
+
+		return await response.json();
+	}
+
+	private async execPagedApiCommand_<Result>(
+		method: 'GET',
+		route: string,
+		params: Record<string, string>,
+		deserializeItem: (data: Json)=> Result,
+	): Promise<Result[]> {
+		const searchParams = new URLSearchParams(params);
+
+		const results: Result[] = [];
+		let hasMore = true;
+		for (let page = 1; hasMore; page++) {
+			searchParams.set('page', String(page));
+			searchParams.set('limit', '10');
+			const response = await this.execApiCommand_(
+				method, `${route}?${searchParams}`,
+			);
+			if (
+				typeof response !== 'object'
+				|| !('has_more' in response)
+				|| !('items' in response)
+				|| !Array.isArray(response.items)
+			) {
+				throw new Error(`Invalid response: ${JSON.stringify(response)}`);
+			}
+			hasMore = !!response.has_more;
+
+			for (const item of response.items) {
+				results.push(deserializeItem(item));
+			}
+		}
+
+		return results;
+	}
+
+	private async decrypt_() {
+		// E2EE decryption can occasionally fail with "Master key is not loaded:".
+		// Allow e2ee decryption to be retried:
+		await retryWithCount(async () => {
+			const result = await this.execCliCommand_('e2ee', 'decrypt', '--force');
+			if (!result.stdout.includes('Completed decryption.')) {
+				throw new Error(`Decryption did not complete: ${result.stdout}`);
+			}
+		}, {
+			count: 3,
+			onFail: async (error)=>{
+				logger.warn('E2EE decryption failed:', error);
+				logger.info('Syncing before retry...');
+				await this.execCliCommand_('sync');
+			},
+		});
+	}
+
+	public async sync() {
+		logger.info('Sync', this.email);
+
+		await this.tracker_.sync();
+
+		const result = await this.execCliCommand_('sync');
+		if (result.stdout.match(/Last error:/i)) {
+			throw new Error(`Sync failed: ${result.stdout}`);
+		}
+
+		await this.decrypt_();
+	}
+
+	public async createFolder(folder: FolderMetadata) {
+		logger.info('Create folder', folder.id, 'in', `${folder.parentId ?? 'root'}/${this.email}`);
+		await this.tracker_.createFolder(folder);
+
+		await this.execApiCommand_('POST', '/folders', {
+			id: folder.id,
+			title: folder.title,
+			parent_id: folder.parentId ?? '',
+		});
+	}
+
+	private async assertNoteMatchesState_(expected: NoteData) {
+		assert.equal(
+			(await this.execCliCommand_('cat', expected.id)).stdout,
+			`${expected.title}\n\n${expected.body}`,
+			'note should exist',
+		);
+	}
+
+	public async createNote(note: NoteData) {
+		logger.info('Create note', note.id, 'in', `${note.parentId}/${this.email}`);
+		await this.tracker_.createNote(note);
+
+		await this.execApiCommand_('POST', '/notes', {
+			id: note.id,
+			title: note.title,
+			body: note.body,
+			parent_id: note.parentId ?? '',
+		});
+		await this.assertNoteMatchesState_(note);
+	}
+
+	public async updateNote(note: NoteData) {
+		logger.info('Update note', note.id, 'in', `${note.parentId}/${this.email}`);
+		await this.tracker_.updateNote(note);
+		await this.execApiCommand_('PUT', `/notes/${encodeURIComponent(note.id)}`, {
+			title: note.title,
+			body: note.body,
+			parent_id: note.parentId ?? '',
+		});
+		await this.assertNoteMatchesState_(note);
+	}
+
+	public async deleteFolder(id: string) {
+		logger.info('Delete folder', id, 'in', this.email);
+		await this.tracker_.deleteFolder(id);
+
+		await this.execCliCommand_('rmbook', '--permanent', '--force', id);
+	}
+
+	public async shareFolder(id: string, shareWith: Client) {
+		await this.tracker_.shareFolder(id, shareWith);
+
+		logger.info('Share', id, 'with', shareWith.email);
+		await this.execCliCommand_('share', 'add', id, shareWith.email);
+		await this.sync();
+		await shareWith.sync();
+
+		const shareWithIncoming = JSON.parse((await shareWith.execCliCommand_('share', 'list', '--json')).stdout);
+		const pendingInvitations = shareWithIncoming.invitations.filter((invitation: unknown) => {
+			if (typeof invitation !== 'object' || !('accepted' in invitation)) {
+				throw new Error('Invalid invitation format');
+			}
+			return !invitation.accepted;
+		});
+		assert.deepEqual(pendingInvitations, [
+			{
+				accepted: false,
+				waiting: true,
+				rejected: false,
+				folderId: id,
+				fromUser: {
+					email: this.email,
+				},
+			},
+		], 'there should be a single incoming share from the expected user');
+
+		await shareWith.execCliCommand_('share', 'accept', id);
+	}
+
+	public async moveItem(itemId: ItemId, newParentId: ItemId) {
+		logger.info('Move', itemId, 'to', newParentId);
+		await this.tracker_.moveItem(itemId, newParentId);
+		const movingToRoot = !newParentId;
+		await this.execCliCommand_('mv', itemId, movingToRoot ? 'root' : newParentId);
+	}
+
+	public async listNotes() {
+		const params = {
+			fields: 'id,parent_id,body,title,is_conflict,conflict_original_id',
+			include_deleted: '1',
+			include_conflicts: '1',
+		};
+		return await this.execPagedApiCommand_(
+			'GET',
+			'/notes',
+			params,
+			item => ({
+				id: getStringProperty(item, 'id'),
+				parentId: getNumberProperty(item, 'is_conflict') === 1 ? (
+					`[conflicts for ${getStringProperty(item, 'conflict_original_id')} in ${this.email}]`
+				) : getStringProperty(item, 'parent_id'),
+				title: getStringProperty(item, 'title'),
+				body: getStringProperty(item, 'body'),
+			}),
+		);
+	}
+
+	public async listFolders() {
+		const params = {
+			fields: 'id,parent_id,title',
+			include_deleted: '1',
+		};
+		return await this.execPagedApiCommand_(
+			'GET',
+			'/folders',
+			params,
+			item => ({
+				id: getStringProperty(item, 'id'),
+				parentId: getStringProperty(item, 'parent_id'),
+				title: getStringProperty(item, 'title'),
+			}),
+		);
+	}
+
+	public async randomFolder(options: RandomFolderOptions) {
+		return this.tracker_.randomFolder(options);
+	}
+
+	public async allFolderDescendants(parentId: ItemId) {
+		return this.tracker_.allFolderDescendants(parentId);
+	}
+
+	public async randomNote() {
+		return this.tracker_.randomNote();
+	}
+
+	public async checkState(_allClients: Client[]) {
+		logger.info('Check state', this.email);
+
+		type ItemSlice = { id: string };
+		const compare = (a: ItemSlice, b: ItemSlice) => {
+			if (a.id === b.id) return 0;
+			return a.id < b.id ? -1 : 1;
+		};
+
+		const assertNoAdjacentEqualIds = (sortedById: ItemSlice[], assertionLabel: string) => {
+			for (let i = 1; i < sortedById.length; i++) {
+				const current = sortedById[i];
+				const previous = sortedById[i - 1];
+				assert.notEqual(
+					current.id,
+					previous.id,
+					`[${assertionLabel}] item ${i} should have a different ID from item ${i - 1}`,
+				);
+			}
+		};
+
+		const checkNoteState = async () => {
+			const notes = [...await this.listNotes()];
+			const expectedNotes = [...await this.tracker_.listNotes()];
+
+			notes.sort(compare);
+			expectedNotes.sort(compare);
+
+			assertNoAdjacentEqualIds(notes, 'notes');
+			assertNoAdjacentEqualIds(expectedNotes, 'expectedNotes');
+			assert.deepEqual(notes, expectedNotes, 'should have the same notes as the expected state');
+		};
+
+		const checkFolderState = async () => {
+			const folders = [...await this.listFolders()];
+			const expectedFolders = [...await this.tracker_.listFolders()];
+
+			folders.sort(compare);
+			expectedFolders.sort(compare);
+
+			assertNoAdjacentEqualIds(folders, 'folders');
+			assertNoAdjacentEqualIds(expectedFolders, 'expectedFolders');
+			assert.deepEqual(folders, expectedFolders, 'should have the same folders as the expected state');
+		};
+
+		await checkNoteState();
+		await checkFolderState();
+	}
+}
+
+export default Client;
+

packages/tools/fuzzer/ClientPool.ts

@@ -0,0 +1,54 @@
+import ActionTracker from './ActionTracker';
+import Client from './Client';
+import { CleanupTask, FuzzContext } from './types';
+
+type AddCleanupTask = (task: CleanupTask)=> void;
+type ClientFilter = (client: Client)=> boolean;
+
+export default class ClientPool {
+	public static async create(
+		context: FuzzContext,
+		clientCount: number,
+		addCleanupTask: AddCleanupTask,
+	) {
+		if (clientCount <= 0) throw new Error('There must be at least 1 client');
+
+		const actionTracker = new ActionTracker(context);
+		const clientPool: Client[] = [];
+		for (let i = 0; i < clientCount; i++) {
+			const client = await Client.create(actionTracker, context);
+			addCleanupTask(() => client.close());
+			clientPool.push(client);
+		}
+
+		return new ClientPool(context, clientPool);
+	}
+	public constructor(
+		private readonly context_: FuzzContext,
+		public readonly clients: Client[],
+	) { }
+
+	public randomClient(filter: ClientFilter = ()=>true) {
+		const clients = this.clients.filter(filter);
+		return clients[
+			this.context_.randInt(0, clients.length)
+		];
+	}
+
+	public async checkState() {
+		for (const client of this.clients) {
+			await client.checkState(this.clients);
+		}
+	}
+
+	public async syncAll() {
+		for (const client of this.clients) {
+			await client.sync();
+		}
+	}
+
+	public helpText() {
+		return this.clients.map(client => client.getHelpText()).join('\n\n');
+	}
+}
+

packages/tools/fuzzer/Server.ts

@@ -0,0 +1,79 @@
+import { join } from 'path';
+import { HttpMethod, Json, UserData } from './types';
+import { packagesDir } from './constants';
+import JoplinServerApi from '@joplin/lib/JoplinServerApi';
+import { Env } from '@joplin/lib/models/Setting';
+import execa = require('execa');
+import { msleep } from '@joplin/utils/time';
+import Logger from '@joplin/utils/Logger';
+
+const logger = Logger.create('Server');
+
+const createApi = async (serverUrl: string, adminAuth: UserData) => {
+	const api = new JoplinServerApi({
+		baseUrl: () => serverUrl,
+		userContentBaseUrl: () => serverUrl,
+		password: () => adminAuth.password,
+		username: () => adminAuth.email,
+		session: ()=>null,
+		env: Env.Dev,
+	});
+	await api.loadSession();
+	return api;
+};
+
+export default class Server {
+	private api_: JoplinServerApi|null = null;
+	private server_: execa.ExecaChildProcess<string>;
+
+	public constructor(
+		private readonly serverUrl_: string,
+		private readonly adminAuth_: UserData,
+	) {
+		const serverDir = join(packagesDir, 'server');
+		const mainEntrypoint = join(serverDir, 'dist', 'app.js');
+		this.server_ = execa.node(mainEntrypoint, [
+			'--env', 'dev',
+		], {
+			env: { JOPLIN_IS_TESTING: '1' },
+			cwd: join(packagesDir, 'server'),
+			stdin: 'ignore', // No stdin
+			// For debugging:
+			// stderr: process.stderr,
+			// stdout: process.stdout,
+		});
+	}
+
+	public async checkConnection() {
+		let lastError;
+		for (let retry = 0; retry < 30; retry++) {
+			try {
+				const response = await fetch(`${this.serverUrl_}api/ping`);
+				if (response.ok) {
+					return true;
+				}
+			} catch (error) {
+				lastError = error;
+			}
+			await msleep(500);
+		}
+		if (lastError) {
+			throw lastError;
+		}
+		return false;
+	}
+
+	public async execApi(method: HttpMethod, route: string, action: Json) {
+		this.api_ ??= await createApi(this.serverUrl_, this.adminAuth_);
+		logger.debug('API EXEC', method, route, action);
+		const result = await this.api_.exec(method, route, {}, action);
+		return result;
+	}
+
+	public async close() {
+		this.server_.cancel();
+		logger.info('Closed the server.');
+	}
+}
+
+

packages/tools/fuzzer/constants.ts

@@ -0,0 +1,5 @@
+import { dirname, join } from 'path';
+
+export const packagesDir = dirname(dirname(__dirname));
+export const cliDirectory = join(packagesDir, 'app-cli');
+

packages/tools/fuzzer/sync-fuzzer.ts

@@ -0,0 +1,362 @@
+import uuid from '@joplin/lib/uuid';
+import { join } from 'path';
+import { exists, mkdir, remove } from 'fs-extra';
+import Setting, { Env } from '@joplin/lib/models/Setting';
+import Logger, { TargetType } from '@joplin/utils/Logger';
+import { waitForCliInput } from '@joplin/utils/cli';
+import Server from './Server';
+import { CleanupTask, FuzzContext } from './types';
+import ClientPool from './ClientPool';
+import retryWithCount from './utils/retryWithCount';
+import Client from './Client';
+import SeededRandom from './utils/SeededRandom';
+import { env } from 'process';
+import yargs = require('yargs');
+import { strict as assert } from 'assert';
+const { shimInit } = require('@joplin/lib/shim-init-node');
+
+const globalLogger = new Logger();
+globalLogger.addTarget(TargetType.Console);
+Logger.initializeGlobalLogger(globalLogger);
+const logger = Logger.create('fuzzer');
+
+const createProfilesDirectory = async () => {
+	const path = join(__dirname, 'profiles-tmp');
+	if (await exists(path)) {
+		throw new Error([
+			'Another instance of the sync fuzzer may be running!',
+			'The parent directory for test profiles already exists. An instance of the fuzzer is either already running or was closed before it could clean up.',
+			`To ignore this issue, delete ${JSON.stringify(path)} and re-run the fuzzer.`,
+		].join('\n'));
+	}
+
+	await mkdir(path);
+	return {
+		path,
+		remove: async () => {
+			await remove(path);
+		},
+	};
+};
+
+const doRandomAction = async (context: FuzzContext, client: Client, clientPool: ClientPool) => {
+	const selectOrCreateParentFolder = async () => {
+		let parentId = (await client.randomFolder({}))?.id;
+
+		// Create a toplevel folder to serve as this
+		// folder's parent if none exist yet
+		if (!parentId) {
+			parentId = uuid.create();
+			await client.createFolder({
+				parentId: '',
+				id: parentId,
+				title: 'Parent folder',
+			});
+		}
+
+		return parentId;
+	};
+
+	const selectOrCreateNote = async () => {
+		let note = await client.randomNote();
+
+		if (!note) {
+			await client.createNote({
+				parentId: await selectOrCreateParentFolder(),
+				id: uuid.create(),
+				title: 'Test note',
+				body: 'Body',
+			});
+
+			note = await client.randomNote();
+			assert.ok(note, 'should have selected a random note');
+		}
+
+		return note;
+	};
+
+	const actions = {
+		newSubfolder: async () => {
+			const folderId = uuid.create();
+			const parentId = await selectOrCreateParentFolder();
+
+			await client.createFolder({
+				parentId: parentId,
+				id: folderId,
+				title: 'Subfolder',
+			});
+
+			return true;
+		},
+		newToplevelFolder: async () => {
+			const folderId = uuid.create();
+			await client.createFolder({
+				parentId: null,
+				id: folderId,
+				title: `Folder ${context.randInt(0, 1000)}`,
+			});
+
+			return true;
+		},
+		newNote: async () => {
+			const parentId = await selectOrCreateParentFolder();
+			await client.createNote({
+				parentId: parentId,
+				title: `Test (x${context.randInt(0, 1000)})`,
+				body: 'Testing...',
+				id: uuid.create(),
+			});
+
+			return true;
+		},
+		renameNote: async () => {
+			const note = await selectOrCreateNote();
+
+			await client.updateNote({
+				...note,
+				title: `Renamed (${context.randInt(0, 1000)})`,
+			});
+
+			return true;
+		},
+		updateNoteBody: async () => {
+			const note = await selectOrCreateNote();
+
+			await client.updateNote({
+				...note,
+				body: `${note.body}\n\nUpdated.\n`,
+			});
+
+			return true;
+		},
+		moveNote: async () => {
+			const note = await client.randomNote();
+			if (!note) return false;
+			const targetParent = await client.randomFolder({
+				filter: folder => folder.id !== note.parentId,
+			});
+			if (!targetParent) return false;
+
+			await client.moveItem(note.id, targetParent.id);
+
+			return true;
+		},
+		shareFolder: async () => {
+			const target = await client.randomFolder({
+				filter: candidate => (
+					!candidate.parentId && !candidate.isShareRoot
+				),
+			});
+			if (!target) return false;
+
+			const other = clientPool.randomClient(c => c !== client);
+			await client.shareFolder(target.id, other);
+			return true;
+		},
+		deleteFolder: async () => {
+			const target = await client.randomFolder({});
+			if (!target) return false;
+
+			await client.deleteFolder(target.id);
+			return true;
+		},
+		moveFolderToToplevel: async () => {
+			const target = await client.randomFolder({
+				// Don't choose items that are already toplevel
+				filter: item => !!item.parentId,
+			});
+			if (!target) return false;
+
+			await client.moveItem(target.id, '');
+			return true;
+		},
+		moveFolderTo: async () => {
+			const target = await client.randomFolder({
+				// Don't move shared folders (should not be allowed by the GUI in the main apps).
+				filter: item => !item.isShareRoot,
+			});
+			if (!target) return false;
+
+			const targetDescendants = new Set(await client.allFolderDescendants(target.id));
+
+			const newParent = await client.randomFolder({
+				filter: (item) => {
+					// Avoid making the folder a child of itself
+					return !targetDescendants.has(item.id);
+				},
+			});
+			if (!newParent) return false;
+
+			await client.moveItem(target.id, newParent.id);
+			return true;
+		},
+	};
+
+	const actionKeys = [...Object.keys(actions)] as (keyof typeof actions)[];
+
+	let result = false;
+	while (!result) { // Loop until an action was done
+		const randomAction = actionKeys[context.randInt(0, actionKeys.length)];
+		logger.info(`Action: ${randomAction} in ${client.email}`);
+		result = await actions[randomAction]();
+		if (!result) {
+			logger.info(`  ${randomAction} was skipped (preconditions not met).`);
+		}
+	}
+};
+
+interface Options {
+	seed: number;
+	maximumSteps: number;
+	maximumStepsBetweenSyncs: number;
+	clientCount: number;
+}
+
+const main = async (options: Options) => {
+	shimInit();
+	Setting.setConstant('env', Env.Dev);
+
+	const cleanupTasks: CleanupTask[] = [];
+
+	const cleanUp = async () => {
+		logger.info('Cleaning up....');
+		while (cleanupTasks.length) {
+			const task = cleanupTasks.pop();
+			try {
+				await task();
+			} catch (error) {
+				logger.warn('Clean up task failed:', error);
+			}
+		}
+	};
+
+	// Run cleanup on Ctrl-C
+	process.on('SIGINT', async () => {
+		logger.info('Intercepted ctrl-c. Cleaning up...');
+		await cleanUp();
+		process.exit(1);
+	});
+
+	let clientHelpText;
+
+	try {
+		const joplinServerUrl = 'http://localhost:22300/';
+		const server = new Server(joplinServerUrl, {
+			email: 'admin@localhost',
+			password: env['FUZZER_SERVER_ADMIN_PASSWORD'] ?? 'admin',
+		});
+		cleanupTasks.push(() => server.close());
+
+		if (!await server.checkConnection()) {
+			throw new Error('Could not connect to the server.');
+		}
+
+		const profilesDirectory = await createProfilesDirectory();
+		cleanupTasks.push(profilesDirectory.remove);
+
+		logger.info('Starting with seed', options.seed);
+		const random = new SeededRandom(options.seed);
+
+		const fuzzContext: FuzzContext = {
+			serverUrl: joplinServerUrl,
+			baseDir: profilesDirectory.path,
+			execApi: server.execApi.bind(server),
+			randInt: (a, b) => random.nextInRange(a, b),
+		};
+		const clientPool = await ClientPool.create(
+			fuzzContext,
+			options.clientCount,
+			task => { cleanupTasks.push(task); },
+		);
+		clientHelpText = clientPool.helpText();
+
+		const maxSteps = options.maximumSteps;
+		for (let stepIndex = 1; maxSteps <= 0 || stepIndex <= maxSteps; stepIndex++) {
+			const client = clientPool.randomClient();
+
+			// Ensure that the client starts up-to-date with the other synced clients.
+			await client.sync();
+
+			logger.info('Step', stepIndex, '/', maxSteps > 0 ? maxSteps : 'Infinity');
+			const actionsBeforeFullSync = fuzzContext.randInt(1, options.maximumStepsBetweenSyncs + 1);
+			for (let subStepIndex = 1; subStepIndex <= actionsBeforeFullSync; subStepIndex++) {
+				if (actionsBeforeFullSync > 1) {
+					logger.info('Sub-step', subStepIndex, '/', actionsBeforeFullSync, '(in step', stepIndex, ')');
+				}
+				await doRandomAction(fuzzContext, client, clientPool);
+			}
+			await client.sync();
+
+			// .checkState can fail occasionally due to incomplete
+			// syncs (perhaps because the server is still processing
+			// share-related changes?). Allow this to be retried:
+			await retryWithCount(async () => {
+				await clientPool.checkState();
+			}, {
+				count: 3,
+				onFail: async () => {
+					logger.info('.checkState failed. Syncing all clients...');
+					await clientPool.syncAll();
+				},
+			});
+		}
+	} catch (error) {
+		logger.error('ERROR', error);
+		if (clientHelpText) {
+			logger.info('Client information:\n', clientHelpText);
+		}
+		logger.info('An error occurred. Pausing before continuing cleanup.');
+		await waitForCliInput();
+		process.exitCode = 1;
+	} finally {
+		await cleanUp();
+
+		logger.info('Cleanup complete');
+		process.exit();
+	}
+};
+
+
+void yargs
+	.usage('$0 <cmd>')
+	.command(
+		'start',
+		[
+			'Starts the synchronization fuzzer. The fuzzer starts Joplin Server, creates multiple CLI clients, and attempts to find sync bugs.\n\n',
+			'The fuzzer starts Joplin Server in development mode, using the existing development mode database and uses the admin@localhost user to',
+			'create and set up user accounts.\n',
+			'Use the FUZZER_SERVER_ADMIN_PASSWORD environment variable to specify the admin@localhost password for this dev version of Joplin Server.\n\n',
+			'If the fuzzer detects incorrect/unexpected client state, it pauses, allowing the profile directories and databases',
+			'of the clients to be inspected.',
+		].join(' '),
+		(yargs) => {
+			return yargs.options({
+				'seed': { type: 'number', default: 12345 },
+				'steps': {
+					type: 'number',
+					default: 0,
+					defaultDescription: 'The maximum number of steps to take before stopping the fuzzer. Set to zero for an unlimited number of steps.',
+				},
+				'steps-between-syncs': {
+					type: 'number',
+					default: 3,
+					defaultDescription: 'The maximum number of sub-steps taken before all clients are synchronised.',
+				},
+				'clients': {
+					type: 'number',
+					default: 3,
+					defaultDescription: 'Number of client apps to create.',
+				},
+			});
+		},
+		async (argv) => {
+			await main({
+				seed: argv.seed,
+				maximumSteps: argv.steps,
+				clientCount: argv.clients,
+				maximumStepsBetweenSyncs: argv['steps-between-syncs'],
+			});
+		},
+	)
+	.help()
+	.argv;

packages/tools/fuzzer/types.ts

@@ -0,0 +1,62 @@
+import type Client from './Client';
+
+export type Json = string|number|Json[]|{ [key: string]: Json };
+
+export type HttpMethod = 'GET'|'POST'|'DELETE'|'PUT'|'PATCH';
+
+export type ItemId = string;
+export type NoteData = {
+	parentId: ItemId;
+	id: ItemId;
+	title: string;
+	body: string;
+};
+export type FolderMetadata = {
+	parentId: ItemId;
+	id: ItemId;
+	title: string;
+};
+export type FolderData = FolderMetadata & {
+	childIds: ItemId[];
+	isShareRoot: boolean;
+};
+export type TreeItem = NoteData|FolderData;
+
+export const isFolder = (item: TreeItem): item is FolderData => {
+	return 'childIds' in item;
+};
+
+export interface FuzzContext {
+	serverUrl: string;
+	baseDir: string;
+	execApi: (method: HttpMethod, route: string, debugAction: Json)=> Promise<Json>;
+	randInt: (low: number, high: number)=> number;
+}
+
+export interface RandomFolderOptions {
+	filter?: (folder: FolderData)=> boolean;
+}
+
+export interface ActionableClient {
+	createFolder(data: FolderMetadata): Promise<void>;
+	shareFolder(id: ItemId, shareWith: Client): Promise<void>;
+	deleteFolder(id: ItemId): Promise<void>;
+	createNote(data: NoteData): Promise<void>;
+	updateNote(data: NoteData): Promise<void>;
+	moveItem(itemId: ItemId, newParentId: ItemId): Promise<void>;
+	sync(): Promise<void>;
+
+	listNotes(): Promise<NoteData[]>;
+	listFolders(): Promise<FolderMetadata[]>;
+	allFolderDescendants(parentId: ItemId): Promise<ItemId[]>;
+	randomFolder(options: RandomFolderOptions): Promise<FolderMetadata>;
+	randomNote(): Promise<NoteData>;
+}
+
+export interface UserData {
+	email: string;
+	password: string;
+}
+
+export type CleanupTask = ()=> Promise<void>;
+

packages/tools/fuzzer/utils/SeededRandom.ts

@@ -0,0 +1,52 @@
+
+// SeededRandom provides a very simple random number generator
+// that can be seeded (since NodeJS built-ins can't).
+//
+// See also:
+// - https://arxiv.org/pdf/1704.00358
+// - https://en.wikipedia.org/wiki/Middle-square_method
+
+// Some large odd number, see https://en.wikipedia.org/wiki/Weyl_sequence
+const step = BigInt('0x12345678ABCDE123'); // uint64
+const maxSize = BigInt(1) << BigInt(64);
+
+const extractMiddle = (value: bigint, halfSize: bigint) => {
+	// Remove the lower quarter
+	const quarterSize = halfSize / BigInt(2);
+	value >>= quarterSize;
+
+	// Remove the upper quarter
+	const halfMaximumValue = BigInt(1) << halfSize;
+	value %= halfMaximumValue;
+
+	return value;
+};
+
+export default class SeededRandom {
+	private value_: bigint;
+	private nextStep_: bigint = step;
+	private halfSize_ = BigInt(32);
+
+	public constructor(seed: number) {
+		this.value_ = BigInt(seed);
+	}
+
+	public next() {
+		this.value_ = this.value_ * this.value_ + this.nextStep_;
+
+		// Move to the next item in the sequence. Mod to prevent from getting
+		// too large. See https://en.wikipedia.org/wiki/Weyl_sequence.
+		this.nextStep_ = (step + this.nextStep_) % maxSize;
+
+		this.value_ = extractMiddle(this.value_, this.halfSize_);
+		return this.value_;
+	}
+
+	// The resultant range includes `a` but excludes `b`.
+	public nextInRange(a: number, b: number) {
+		if (b <= a + 1) return a;
+
+		const range = b - a;
+		return Number(this.next() % BigInt(range)) + a;
+	}
+}

packages/tools/fuzzer/utils/getNumberProperty.ts

@@ -0,0 +1,11 @@
+import getProperty from './getProperty';
+
+const getNumberProperty = (object: unknown, propertyName: string) => {
+	const value = getProperty(object, propertyName);
+	if (typeof value !== 'number') {
+		throw new Error(`Property value is not a string (is ${typeof value})`);
+	}
+	return value;
+};
+
+export default getNumberProperty;

packages/tools/fuzzer/utils/getProperty.ts

@@ -0,0 +1,15 @@
+
+const getProperty = (object: unknown, propertyName: string) => {
+	if (typeof object !== 'object' || object === null) {
+		throw new Error(`Cannot access property ${JSON.stringify(propertyName)} on non-object`);
+	}
+
+	if (!(propertyName in object)) {
+		throw new Error(`No such property ${JSON.stringify(propertyName)} in object`);
+	}
+
+	return object[propertyName as keyof object];
+};
+
+export default getProperty;
+

packages/tools/fuzzer/utils/getStringProperty.ts

@@ -0,0 +1,11 @@
+import getProperty from './getProperty';
+
+const getStringProperty = (object: unknown, propertyName: string) => {
+	const value = getProperty(object, propertyName);
+	if (typeof value !== 'string') {
+		throw new Error(`Property value is not a string (is ${typeof value})`);
+	}
+	return value;
+};
+
+export default getStringProperty;

packages/tools/fuzzer/utils/retryWithCount.ts

@@ -0,0 +1,21 @@
+interface Options {
+	count: number;
+	onFail: (error: Error)=> Promise<void>;
+}
+
+const retryWithCount = async (task: ()=> Promise<void>, { count, onFail }: Options) => {
+	let lastError: Error|null = null;
+	for (let retry = 0; retry < count; retry ++) {
+		try {
+			return await task();
+		} catch (error) {
+			await onFail(error);
+			lastError = error;
+		}
+	}
+
+	if (lastError) throw lastError;
+};
+
+export default retryWithCount;
+

packages/utils/cli.ts

@@ -1,16 +1,17 @@
-
 const readline = require('readline/promises');
 
 /* eslint-disable no-console */
 
+export const isTTY = () => process.stdin.isTTY;
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
 let readlineInterface: any = null;
-const waitForCliInput = async () => {
+export const waitForCliInput = async () => {
 	readlineInterface ??= readline.createInterface({
 		input: process.stdin,
 		output: process.stdout,
 	});
-	if (process.stdin.isTTY) {
+	if (isTTY()) {
 		const green = '\x1b[92m';
 		const reset = '\x1b[0m';
 		await readlineInterface.question(`${green}[Press enter to continue]${reset}`);
@@ -21,4 +22,3 @@ const waitForCliInput = async () => {
 	}
 };
 
-export default waitForCliInput;

packages/utils/package.json

@@ -18,7 +18,8 @@
     "./types": "./dist/types.js",
     "./url": "./dist/url.js",
     "./ipc": "./dist/ipc.js",
-    "./path": "./dist/path.js"
+    "./path": "./dist/path.js",
+    "./cli": "./dist/cli.js"
   },
   "publishConfig": {
     "access": "public"