Chore: Sync fuzzer: Test adding, removing resources from notes (#14185)

.eslintignore

@@ -1851,17 +1851,22 @@ packages/tools/fuzzer/ClientPool.js
 packages/tools/fuzzer/Server.js
 packages/tools/fuzzer/constants.js
 packages/tools/fuzzer/model/FolderRecord.js
+packages/tools/fuzzer/model/ResourceRecord.js
 packages/tools/fuzzer/sync-fuzzer.js
 packages/tools/fuzzer/types.js
 packages/tools/fuzzer/utils/ProgressBar.js
 packages/tools/fuzzer/utils/SeededRandom.js
 packages/tools/fuzzer/utils/diffSortedStringArrays.test.js
 packages/tools/fuzzer/utils/diffSortedStringArrays.js
+packages/tools/fuzzer/utils/extractResourceIds.js
 packages/tools/fuzzer/utils/getNumberProperty.js
 packages/tools/fuzzer/utils/getProperty.js
 packages/tools/fuzzer/utils/getStringProperty.js
+packages/tools/fuzzer/utils/hangingIndent.js
 packages/tools/fuzzer/utils/logDiffDebug.js
 packages/tools/fuzzer/utils/openDebugSession.js
+packages/tools/fuzzer/utils/randomId.test.js
+packages/tools/fuzzer/utils/randomId.js
 packages/tools/fuzzer/utils/randomString.js
 packages/tools/fuzzer/utils/retryWithCount.js
 packages/tools/generate-database-types.js

.gitignore

@@ -1825,17 +1825,22 @@ packages/tools/fuzzer/ClientPool.js
 packages/tools/fuzzer/Server.js
 packages/tools/fuzzer/constants.js
 packages/tools/fuzzer/model/FolderRecord.js
+packages/tools/fuzzer/model/ResourceRecord.js
 packages/tools/fuzzer/sync-fuzzer.js
 packages/tools/fuzzer/types.js
 packages/tools/fuzzer/utils/ProgressBar.js
 packages/tools/fuzzer/utils/SeededRandom.js
 packages/tools/fuzzer/utils/diffSortedStringArrays.test.js
 packages/tools/fuzzer/utils/diffSortedStringArrays.js
+packages/tools/fuzzer/utils/extractResourceIds.js
 packages/tools/fuzzer/utils/getNumberProperty.js
 packages/tools/fuzzer/utils/getProperty.js
 packages/tools/fuzzer/utils/getStringProperty.js
+packages/tools/fuzzer/utils/hangingIndent.js
 packages/tools/fuzzer/utils/logDiffDebug.js
 packages/tools/fuzzer/utils/openDebugSession.js
+packages/tools/fuzzer/utils/randomId.test.js
+packages/tools/fuzzer/utils/randomId.js
 packages/tools/fuzzer/utils/randomString.js
 packages/tools/fuzzer/utils/retryWithCount.js
 packages/tools/generate-database-types.js

packages/tools/fuzzer/ActionRunner.ts

@@ -1,7 +1,6 @@
-import uuid from '@joplin/lib/uuid';
 import Client from './Client';
 import ClientPool from './ClientPool';
-import { assertIsFolder, assertIsNote, FuzzContext, ItemId, RandomFolderOptions } from './types';
+import { assertIsFolder, assertIsNote, FuzzContext, ItemId, RandomFolderOptions, ResourceData } from './types';
 import { strict as assert } from 'assert';
 import Logger from '@joplin/utils/Logger';
 import retryWithCount from './utils/retryWithCount';
@@ -31,10 +30,12 @@ export default class ActionRunner {
 			await this.clientPool_.checkState();
 		}, {
 			count: 4,
-			delayOnFailure: count => count * Second * 2,
-			onFail: async () => {
-				logger.info('.checkState failed. Syncing all clients...');
-				await this.clientPool_.syncAll();
+			delayOnFailure: count => count * Second * 3,
+			onFail: async ({ willRetry }) => {
+				if (willRetry) {
+					logger.info('.checkState failed. Syncing all clients...');
+					await this.clientPool_.syncAll();
+				}
 			},
 		});
 	}
@@ -148,7 +149,7 @@ const getActions = (context: FuzzContext, clientPool: ClientPool, client: Client
 		// Create a toplevel folder to serve as this
 		// folder's parent if none exist yet
 		if (!parentId) {
-			parentId = uuid.create();
+			parentId = context.randomId();
 			await client.createFolder({
 				parentId: '',
 				id: parentId,
@@ -171,7 +172,7 @@ const getActions = (context: FuzzContext, clientPool: ClientPool, client: Client
 			await client.createNote({
 				...defaultNoteProperties,
 				parentId: await selectOrCreateWriteableFolder(),
-				id: uuid.create(),
+				id: context.randomId(),
 				title: 'Test note',
 				body: 'Body',
 			});
@@ -184,12 +185,16 @@ const getActions = (context: FuzzContext, clientPool: ClientPool, client: Client
 	};
 
 	const noteById = (id: ItemId) => {
+		assert.ok(client.itemExists(id), `Could not find note with ID ${id} in client ${client.email}'s expected state.`);
+
 		const note = client.itemById(id);
 		assertIsNote(note);
 		return note;
 	};
 
 	const folderById = (id: ItemId) => {
+		assert.ok(client.itemExists(id), `Could not find folder with ID ${id} in client ${client.email}'s expected state.`);
+
 		const folder = client.itemById(id);
 		assertIsFolder(folder);
 		return folder;
@@ -244,14 +249,29 @@ const getActions = (context: FuzzContext, clientPool: ClientPool, client: Client
 
 	addAction('updateNoteBody', async ({ id }) => {
 		const note = noteById(id);
+
 		await client.updateNote({
 			...note,
-			body: `${note.body}\n\nUpdated.\n`,
+			body: `${note.body}\n\nUpdated!`,
 		});
 
 		return true;
 	}, { id: selectOrCreateWriteableNote });
 
+	addAction('attachResourceTo', async ({ noteId, resourceId }) => {
+		const resourceData: ResourceData = {
+			id: resourceId,
+			mimeType: 'text/plain',
+			title: 'Test!',
+		};
+		await client.attachResource(noteById(noteId), resourceData);
+
+		return true;
+	}, {
+		noteId: selectOrCreateWriteableNote,
+		resourceId: () => context.randomId(),
+	});
+
 	addAction('moveNote', async ({ noteId, targetFolderId }) => {
 		const note = noteById(noteId);
 		const newParent = await folderByIdOrRandom(targetFolderId, {
@@ -267,6 +287,19 @@ const getActions = (context: FuzzContext, clientPool: ClientPool, client: Client
 		targetFolderId: undefinedId,
 	});
 
+	addAction('duplicateNote', async ({ id, newNoteId }) => {
+		const note = noteById(id);
+
+		await client.createNote({
+			...note,
+			id: newNoteId,
+		});
+		return true;
+	}, {
+		id: selectOrCreateWriteableNote,
+		newNoteId: () => context.randomId(),
+	});
+
 	addAction('deleteNote', async ({ id }) => {
 		const validatedNote = noteById(id); // Ensure, e.g., that the note exists
 
@@ -420,8 +453,12 @@ const getActions = (context: FuzzContext, clientPool: ClientPool, client: Client
 		}, {
 			delayOnFailure: (count) => Second * count,
 			count: 3,
-			onFail: async (error) => {
-				logger.warn('other.sync/other.checkState failed with', error, 'retrying...');
+			onFail: async ({ error, willRetry }) => {
+				logger.warn(
+					'other.sync/other.checkState failed with',
+					error,
+					willRetry ? 'retrying...' : '',
+				);
 			},
 		});
 

packages/tools/fuzzer/ActionTracker.ts

@@ -1,6 +1,8 @@
 import { strict as assert } from 'assert';
-import { ActionableClient, FolderData, FuzzContext, ItemId, NoteData, ShareOptions, TreeItem, assertIsFolder, isFolder } from './types';
+import { ActionableClient, FolderData, FuzzContext, ItemId, NoteData, ShareOptions, TreeItem, assertIsFolder, isFolder, isNote, isResource } from './types';
 import FolderRecord from './model/FolderRecord';
+import { extractResourceUrls } from '@joplin/lib/urlUtils';
+import ResourceRecord from './model/ResourceRecord';
 
 interface ClientData {
 	childIds: ItemId[];
@@ -68,44 +70,71 @@ class ActionTracker {
 	}
 
 	private checkRep_() {
+		const checkParentId = (item: TreeItem) => {
+			if (item.parentId) {
+				const parent = this.idToItem_.get(item.parentId);
+				assert.ok(parent, `should find parent (id: ${item.parentId})`);
+
+				assert.ok(isFolder(parent), 'parent should be a folder');
+				assert.ok(parent.childIds.includes(item.id), 'parent should include the current item in its children');
+			}
+		};
+		const checkFolder = (folder: FolderRecord) => {
+			for (const childId of folder.childIds) {
+				checkItem(childId);
+			}
+
+			// Shared folders
+			assert.ok(folder.ownedByEmail, 'all folders should have a "shareOwner" property (even if not shared)');
+			if (folder.isRootSharedItem) {
+				assert.equal(folder.parentId, '', 'only toplevel folders should be shared');
+			}
+			for (const sharedWith of folder.shareRecipients) {
+				assert.ok(this.tree_.has(sharedWith), 'all sharee users should exist');
+			}
+			// isSharedWith is only valid for toplevel folders
+			if (folder.parentId === '') {
+				assert.ok(!folder.isSharedWith(folder.ownedByEmail), 'the share owner should not be in an item\'s sharedWith list');
+			}
+
+			// Uniqueness
+			assert.equal(
+				folder.childIds.length,
+				[...new Set(folder.childIds)].length,
+				'child IDs should be unique',
+			);
+		};
+		const checkNote = (note: NoteData) => {
+			assert.ok(!isFolder(note));
+			assert.ok(!isResource(note));
+		};
+		const checkResource = (resource: ResourceRecord) => {
+			assert.ok(!isFolder(resource));
+			assert.ok(!isNote(resource));
+			assert.ok(isResource(resource));
+
+			// References list should be up-to-date
+			for (const noteId of resource.referencedBy) {
+				const note = this.idToItem_.get(noteId);
+				assert.ok(note, `all references should exist (testing ID ${noteId})`);
+				assert.ok(isNote(note), 'all references should be notes');
+				assert.ok(note.body.includes(resource.id), 'all references should include the resource ID');
+			}
+		};
 		const checkItem = (itemId: ItemId) => {
 			assert.match(itemId, /^[a-zA-Z0-9]{32}$/, 'item IDs should be 32 character alphanumeric strings');
 
 			const item = this.idToItem_.get(itemId);
 			assert.ok(!!item, `should find item with ID ${itemId}`);
 
-			if (item.parentId) {
-				const parent = this.idToItem_.get(item.parentId);
-				assert.ok(parent, `should find parent (id: ${item.parentId})`);
-
-				assert.ok(isFolder(parent), 'parent should be a folder');
-				assert.ok(parent.childIds.includes(itemId), 'parent should include the current item in its children');
-			}
+			checkParentId(item);
 
 			if (isFolder(item)) {
-				for (const childId of item.childIds) {
-					checkItem(childId);
-				}
-
-				// Shared folders
-				assert.ok(item.ownedByEmail, 'all folders should have a "shareOwner" property (even if not shared)');
-				if (item.isRootSharedItem) {
-					assert.equal(item.parentId, '', 'only toplevel folders should be shared');
-				}
-				for (const sharedWith of item.shareRecipients) {
-					assert.ok(this.tree_.has(sharedWith), 'all sharee users should exist');
-				}
-				// isSharedWith is only valid for toplevel folders
-				if (item.parentId === '') {
-					assert.ok(!item.isSharedWith(item.ownedByEmail), 'the share owner should not be in an item\'s sharedWith list');
-				}
-
-				// Uniqueness
-				assert.equal(
-					item.childIds.length,
-					[...new Set(item.childIds)].length,
-					'child IDs should be unique',
-				);
+				checkFolder(item);
+			} else if (isNote(item)) {
+				checkNote(item);
+			} else {
+				checkResource(item);
 			}
 		};
 
@@ -263,6 +292,8 @@ class ActionTracker {
 
 					removeItemRecursive(childId);
 				}
+			} else if (isNote(item)) {
+				updateResourceReferences(item, { ...item, body: '' });
 			}
 		};
 		const mapItems = <T> (map: (item: TreeItem)=> T, startFolder?: FolderRecord) => {
@@ -282,6 +313,16 @@ class ActionTracker {
 						workList.push(childId);
 					}
 				}
+				if (isNote(item)) {
+					// Map linked resources
+					const linkedIds = extractResourceUrls(item.body);
+					for (const id of linkedIds) {
+						const item = this.idToItem_.get(id.itemId);
+						if (!item || !isResource(item)) continue;
+
+						result.push(map(item));
+					}
+				}
 			}
 
 			return result;
@@ -342,6 +383,50 @@ class ActionTracker {
 			this.checkRep_();
 		};
 
+		const updateResourceReferences = (noteBefore: NoteData|null, noteAfter: NoteData|null) => {
+			assert.ok(!!noteBefore || !!noteAfter, 'at least one of (noteBefore, noteAfter) must be specified');
+			if (noteBefore && noteAfter) {
+				assert.equal(noteBefore.id, noteAfter.id, 'changing note IDs is not supported');
+			}
+
+			const bodyBefore = noteBefore?.body ?? '';
+			const bodyAfter = noteAfter?.body ?? '';
+			if (bodyBefore === bodyAfter) return;
+
+			const id = noteBefore?.id ?? noteAfter?.id;
+
+			const referencesBefore = extractResourceUrls(bodyBefore).map(r => r.itemId);
+			const referencesAfter = extractResourceUrls(bodyAfter).map(r => r.itemId);
+
+			const newReferences = new Set(referencesAfter);
+			for (const reference of referencesBefore) {
+				newReferences.delete(reference);
+			}
+
+			const removedReferences = new Set(referencesBefore);
+			for (const reference of referencesAfter) {
+				removedReferences.delete(reference);
+			}
+
+			for (const reference of newReferences) {
+				const item = this.idToItem_.get(reference);
+				if (item && isResource(item)) {
+					updateItem(item.id, item.withReference(id), `referenced by ${id}`);
+				}
+			}
+
+			for (const reference of removedReferences) {
+				const item = this.idToItem_.get(reference);
+				if (item && isResource(item)) {
+					updateItem(
+						item.id,
+						item.withoutReference(id),
+						`dereferenced by ${id}`,
+					);
+				}
+			}
+		};
+
 		const tracker: ActionableClient = {
 			createNote: (data: NoteData) => {
 				assertWriteable(data.parentId);
@@ -350,8 +435,9 @@ class ActionTracker {
 				assert.ok(!this.idToItem_.has(data.id), `note ${data.id} should not yet exist`);
 				updateItem(data.id, {
 					...data,
-				}, 'created');
+				}, `created in ${data.parentId}`);
 				addChild(data.parentId, data.id);
+				updateResourceReferences(null, data);
 
 				this.checkRep_();
 				return Promise.resolve();
@@ -364,7 +450,7 @@ class ActionTracker {
 				assert.ok(!!data.parentId, `note ${data.id} should have a parentId`);
 
 				// Additional debugging information about what changed:
-				const changedFieldsInfo = Object.entries(data)
+				const changedFields = Object.entries(data)
 					.filter(([key, newValue]) => {
 						const itemKey = key as keyof NoteData;
 						// isShared is a virtual property
@@ -378,12 +464,39 @@ class ActionTracker {
 				removeChild(oldItem.parentId, data.id);
 				updateItem(data.id, {
 					...data,
-				}, `updated (changed fields: ${JSON.stringify(changedFieldsInfo)})`);
+				}, `updated (changed fields: ${JSON.stringify(changedFields)})`);
 				addChild(data.parentId, data.id);
+				updateResourceReferences(oldItem, data);
 
 				this.checkRep_();
 				return Promise.resolve();
 			},
+			attachResource: async (note, resource) => {
+				const resourceMarkup = `[resource](:/${resource.id})`;
+				const withAttached = { ...note, body: `${note.body}${resourceMarkup}` };
+
+				if (!tracker.itemExists(resource.id)) {
+					await tracker.createResource(resource);
+				}
+				await tracker.updateNote(withAttached);
+				return withAttached;
+			},
+			createResource: async (resource) => {
+				if (tracker.itemExists(resource.id)) {
+					// Don't double-create the item.
+					return Promise.resolve();
+				}
+
+				updateItem(
+					resource.id, new ResourceRecord({
+						...resource,
+						referencedBy: [],
+					}),
+					'created',
+				);
+				this.checkRep_();
+				return Promise.resolve();
+			},
 			createFolder: (data: FolderData) => {
 				const parentId = data.parentId ?? '';
 				assertWriteable(parentId);
@@ -395,7 +508,7 @@ class ActionTracker {
 					sharedWith: [],
 					ownedByEmail: clientId,
 					isShared: false,
-				}), 'created');
+				}), `created ${data.parentId ? `in ${data.parentId}` : '(toplevel)'}`);
 				addChild(data.parentId, data.id);
 
 				this.checkRep_();
@@ -419,7 +532,7 @@ class ActionTracker {
 
 				const item = this.idToItem_.get(id);
 				if (!item) throw new Error(`Not found ${id}`);
-				assert.ok(!isFolder(item), 'should be a note');
+				assert.ok(isNote(item), 'should be a note');
 				assertWriteable(item);
 
 				removeItemRecursive(id);
@@ -480,6 +593,7 @@ class ActionTracker {
 			},
 			moveItem: (itemId, newParentId) => {
 				const item = this.idToItem_.get(itemId);
+				assert.ok(isFolder(item) || isNote(item), `item with ${itemId} should be a folder or a note`);
 
 				const validateParameters = () => {
 					assert.ok(item, `item with ${itemId} should exist`);
@@ -514,10 +628,9 @@ class ActionTracker {
 			publishNote: (id) => {
 				const oldItem = this.idToItem_.get(id);
 				assert.ok(oldItem, 'should exist');
-				assert.ok(!isFolder(oldItem), 'folders cannot be published');
+				assert.ok(isNote(oldItem), 'only notes can be published');
 				assert.ok(!oldItem.published, 'should not be published');
 
-
 				updateItem(id, {
 					...oldItem,
 					published: true,
@@ -529,7 +642,7 @@ class ActionTracker {
 			unpublishNote: (id) => {
 				const oldItem = this.idToItem_.get(id);
 				assert.ok(oldItem, 'should exist');
-				assert.ok(!isFolder(oldItem), 'folders cannot be unpublished');
+				assert.ok(isNote(oldItem), 'only notes can be unpublished');
 				assert.ok(oldItem.published, 'should be published');
 
 				updateItem(id, {
@@ -541,9 +654,15 @@ class ActionTracker {
 				return Promise.resolve();
 			},
 			sync: () => Promise.resolve(),
+			listResources: () => {
+				const items = mapItems(item => {
+					return !isResource(item) ? null : item;
+				}).filter(item => !!item && item.referenceCount > 0);
+				return Promise.resolve(items);
+			},
 			listNotes: () => {
 				const notes = mapItems(item => {
-					return isFolder(item) ? null : item;
+					return !isNote(item) ? null : item;
 				}).filter(item => !!item).map(item => ({
 					...item,
 					isShared: isShared(item),
@@ -597,8 +716,11 @@ class ActionTracker {
 
 				return folders.length ? this.context_.randomFrom(folders) : null;
 			},
-			randomNote: async () => {
-				const notes = await tracker.listNotes();
+			randomNote: async (options) => {
+				let notes = await tracker.listNotes();
+				if (!options.includeReadOnly) {
+					notes = notes.filter(note => !isReadOnly(note.id));
+				}
 				const noteIndex = this.context_.randInt(0, notes.length);
 				return notes.length ? notes[noteIndex] : null;
 			},
@@ -609,6 +731,18 @@ class ActionTracker {
 				if (!item) throw new Error(`No item found with ID ${id}`);
 				return item;
 			},
+			itemExists: (id: ItemId) => {
+				const item = this.idToItem_.get(id);
+				if (!item) return false;
+				if (isResource(item)) return true;
+
+				const root = this.getToplevelParent_(id);
+				if (isFolder(root)) {
+					return root.ownedByEmail === client.email || root.isSharedWith(client.email);
+				}
+
+				return this.tree_.get(clientId).childIds.includes(id);
+			},
 		};
 		return tracker;
 	}

packages/tools/fuzzer/Client.ts

@@ -1,5 +1,5 @@
 import uuid, { createSecureRandom } from '@joplin/lib/uuid';
-import { ActionableClient, FolderData, FuzzContext, HttpMethod, ItemId, Json, NoteData, RandomFolderOptions, RandomNoteOptions, ShareOptions } from './types';
+import { ActionableClient, assertIsNote, FolderData, FuzzContext, HttpMethod, ItemId, Json, NoteData, RandomFolderOptions, RandomNoteOptions, ResourceData, ShareOptions } from './types';
 import { join } from 'path';
 import { mkdir, remove } from 'fs-extra';
 import getStringProperty from './utils/getStringProperty';
@@ -14,7 +14,6 @@ import getNumberProperty from './utils/getNumberProperty';
 import retryWithCount from './utils/retryWithCount';
 import resolvePathWithinDir from '@joplin/lib/utils/resolvePathWithinDir';
 import { formatMsToDateTimeLocal, msleep, Second } from '@joplin/utils/time';
-import shim from '@joplin/lib/shim';
 import { spawn } from 'child_process';
 import AsyncActionQueue from '@joplin/lib/AsyncActionQueue';
 import { createInterface } from 'readline/promises';
@@ -23,6 +22,9 @@ import ProgressBar from './utils/ProgressBar';
 import logDiffDebug from './utils/logDiffDebug';
 import { NoteEntity } from '@joplin/lib/services/database/types';
 import diffSortedStringArrays from './utils/diffSortedStringArrays';
+import extractResourceIds from './utils/extractResourceIds';
+import { substrWithEllipsis } from '@joplin/lib/string-utils';
+import hangingIndent from './utils/hangingIndent';
 
 const logger = Logger.create('Client');
 
@@ -105,6 +107,12 @@ interface CreateRandomItemOptions extends CreateOrUpdateOptions {
 	quiet?: boolean;
 }
 
+class ApiResponseError extends Error {
+	public constructor(public readonly code: number, message: string) {
+		super(message);
+	}
+}
+
 class Client implements ActionableClient {
 	public readonly email: string;
 
@@ -122,7 +130,7 @@ class Client implements ActionableClient {
 	}
 
 	private static async fromAccount(account: AccountData, actionTracker: ActionTracker, context: FuzzContext) {
-		const id = uuid.create();
+		const id = context.randomId();
 		const profileDirectory = join(context.baseDir, id);
 		await mkdir(profileDirectory);
 
@@ -242,6 +250,14 @@ class Client implements ActionableClient {
 
 	private closed_ = false;
 	public async close() {
+		if (this.closed_) {
+			// This can happen if:
+			// - Multiple cleanup callbacks are registered for the client.
+			// - The client was manually closed, but also has a cleanup callback registered.
+			logger.info('Client', this.clientLabel_, 'already closed. Skipping.');
+			return;
+		}
+
 		assert.ok(!this.closed_, 'should not be closed');
 
 		await this.account_.onClientDisconnected();
@@ -388,22 +404,24 @@ class Client implements ActionableClient {
 	// eslint-disable-next-line no-dupe-class-members -- This is not a duplicate class member
 	private async execApiCommand_(method: 'GET', route: string): Promise<string>;
 	// eslint-disable-next-line no-dupe-class-members -- This is not a duplicate class member
-	private async execApiCommand_(method: 'POST'|'PUT', route: string, data: Json): Promise<string>;
+	private async execApiCommand_(method: 'POST'|'PUT', route: string, data: Json|FormData): Promise<string>;
 	// eslint-disable-next-line no-dupe-class-members -- This is not a duplicate class member
-	private async execApiCommand_(method: HttpMethod, route: string, data: Json|null = null): Promise<string> {
+	private async execApiCommand_(method: HttpMethod, route: string, data: Json|FormData|null = null): Promise<string> {
 		route = route.replace(/^[/]/, '');
 		const url = new URL(`http://localhost:${this.apiData_.port}/${route}`);
 		url.searchParams.append('token', this.apiData_.token);
 
 		this.transcript_.push(`\n[[${method} ${url}; body: ${JSON.stringify(data)}]]\n`);
 
-		const response = await shim.fetch(url.toString(), {
+		const response = await fetch(url.toString(), {
 			method,
-			...(data ? { body: JSON.stringify(data) } : undefined),
+			...(data ? {
+				body: data instanceof FormData ? data : JSON.stringify(data),
+			} : undefined),
 		});
 
 		if (!response.ok) {
-			throw new Error(`Request to ${route} failed with error: ${await response.text()}`);
+			throw new ApiResponseError(response.status, `Request to ${route} failed with error: ${await response.text()}`);
 		}
 
 		return await response.text();
@@ -468,11 +486,114 @@ class Client implements ActionableClient {
 			// Certain sync failures self-resolve after a background task is allowed to
 			// run. Delay:
 			delayOnFailure: retry => retry * Second * 2,
-			onFail: async (error) => {
+			onFail: async ({ error, willRetry }) => {
 				logger.debug('Sync error: ', error);
-				logger.info('Sync failed. Retrying...');
+				if (willRetry) {
+					logger.info('Sync failed. Retrying...');
+				}
 			},
 		});
+
+		await this.handleResourceIdChanges_();
+	}
+
+	// Joplin occasionally changes the ID of a resource. Handle this here.
+	// Assumes that the client is up-to-date with the server.
+	private async handleResourceIdChanges_() {
+		type UntrackedAttachment = {
+			id: ItemId;
+			linkedNotes: Set<ItemId>;
+		};
+		const collectUntrackedAttachments = async () => {
+			// Maps from untracked item IDs to the notes that contain that item.
+			const untrackedItemsById = new Map<ItemId, UntrackedAttachment>();
+			const noteActualStates = new Map<ItemId, NoteData>();
+			for (const note of await this.listNotes()) {
+				// Skip notes that are not yet in the expected state. It's possible
+				// that these notes still need to be synced by another client. If so,
+				// attachments in these notes will be processed later:
+				if (!this.tracker_.itemExists(note.id)) continue;
+
+				for (const itemId of extractResourceIds(note.body)) {
+					if (this.tracker_.itemExists(itemId)) continue;
+
+					const noteIds = untrackedItemsById.get(itemId);
+					if (noteIds) {
+						noteIds.linkedNotes.add(note.id);
+					} else {
+						untrackedItemsById.set(itemId, {
+							id: itemId,
+							linkedNotes: new Set([note.id]),
+						});
+					}
+					noteActualStates.set(note.id, note);
+				}
+			}
+
+			return { untrackedItemsById, noteActualStates };
+		};
+
+		const fetchResourceData = async (resourceId: ItemId) => {
+			try {
+				const resourceJson = JSON.parse(
+					await this.execApiCommand_('GET', `/resources/${resourceId}?fields=id,title,mime`),
+				);
+				const resourceData: ResourceData = {
+					id: getStringProperty(resourceJson, 'id'),
+					mimeType: getStringProperty(resourceJson, 'mime'),
+					title: getStringProperty(resourceJson, 'title'),
+				};
+				return resourceData;
+			} catch (error) {
+				if (error instanceof ApiResponseError && error.code === 404) {
+					return null;
+				} else {
+					throw error;
+				}
+			}
+		};
+
+		const removeResourceIds = (text: string) => {
+			for (const id of extractResourceIds(text)) {
+				text = text.split(id).join('');
+			}
+			return text;
+		};
+
+		const textsMatchIgnoringResources = (actual: string, expected: string) => {
+			return removeResourceIds(expected) === removeResourceIds(actual);
+		};
+
+		const { untrackedItemsById, noteActualStates } = await collectUntrackedAttachments();
+
+		for (const { id: resourceId, linkedNotes } of untrackedItemsById.values()) {
+			const resourceData = await fetchResourceData(resourceId);
+			if (!resourceData) {
+				logger.warn('Resource not found:', resourceId);
+				continue;
+			}
+
+			await this.createResource(resourceData);
+			for (const id of linkedNotes) {
+				const expected = this.tracker_.itemById(id);
+				assertIsNote(expected);
+				const actual = noteActualStates.get(id);
+				assertIsNote(actual);
+
+				if (textsMatchIgnoringResources(actual.body, expected.body)) {
+					const firstMatchIndex = actual.body.indexOf(resourceId);
+					// This relies on the fact that **all** resource IDs are length-32 strings:
+					const originalId = expected.body.substring(firstMatchIndex, firstMatchIndex + 32);
+
+					logger.info('Resource rewrite: Updating note', id, ': Replacing', originalId, 'with', resourceId);
+
+					await this.tracker_.updateNote({
+						...expected,
+						body: expected.body.split(originalId).join(resourceId),
+					});
+				}
+			}
+		}
 	}
 
 	public async createOrUpdateMany(actionCount: number) {
@@ -530,7 +651,7 @@ class Client implements ActionableClient {
 		const titleLength = this.context_.randInt(1, 128);
 		const folder = {
 			parentId: parentId,
-			id: id ?? uuid.create(),
+			id: id ?? this.context_.randomId(),
 			title: this.context_.randomString(titleLength).replace(/\n/g, ' '),
 		};
 
@@ -609,7 +730,7 @@ class Client implements ActionableClient {
 			parentId,
 			title: this.context_.randomString(titleLength),
 			body: this.context_.randomString(bodyLength),
-			id: id ?? uuid.create(),
+			id: id ?? this.context_.randomId(),
 		}, { quiet });
 	}
 
@@ -652,6 +773,52 @@ class Client implements ActionableClient {
 		await this.execCliCommand_('rmnote', '--permanent', '--force', id);
 	}
 
+	public async attachResource(note: NoteData, resource: ResourceData): Promise<NoteData> {
+		logger.info('Attach resource', resource.id, 'to note', note.id);
+		const updatedNote = await this.tracker_.attachResource(note, resource);
+
+		await this.execApiCommand_('PUT', `/notes/${encodeURIComponent(note.id)}`, {
+			title: updatedNote.title,
+			body: updatedNote.body,
+			parent_id: updatedNote.parentId ?? '',
+		});
+
+		// Create the resource on the client *after* attaching it to the note so that the
+		// resource is always referenced by at least one note:
+		await this.createResource(resource);
+
+		await this.assertNoteMatchesState_(updatedNote);
+		return updatedNote;
+	}
+
+	public async createResource(resource: ResourceData): Promise<void> {
+		await this.tracker_.createResource(resource);
+
+		const checkExists = async () => {
+			try {
+				await this.execApiCommand_('GET', `/resources/${resource.id}`);
+				return true;
+			} catch (error) {
+				if (error instanceof ApiResponseError && error.code === 404) {
+					return false;
+				}
+				throw error;
+			}
+		};
+
+		if (!await checkExists()) {
+			const resourceForm = new FormData();
+			resourceForm.append('data', new Blob(['test'], { type: resource.mimeType }));
+			resourceForm.append('props', JSON.stringify({
+				title: resource.title,
+				id: resource.id,
+				mime: resource.mimeType,
+			}));
+
+			await this.execApiCommand_('POST', '/resources', resourceForm);
+		}
+	}
+
 	public async deleteFolder(id: string) {
 		logger.info('Delete folder', id, 'in', this.label);
 		await this.tracker_.deleteFolder(id);
@@ -698,8 +865,8 @@ class Client implements ActionableClient {
 		}, {
 			count: 2,
 			delayOnFailure: count => count * Second,
-			onFail: (error)=>{
-				logger.warn('Share failed:', error);
+			onFail: ({ error, willRetry })=>{
+				logger.warn('Share failed:', error, willRetry ? 'Retrying...' : '');
 			},
 		});
 
@@ -755,6 +922,24 @@ class Client implements ActionableClient {
 		await this.execCliCommand_('mv', itemId, movingToRoot ? 'root' : newParentId);
 	}
 
+	public async listResources() {
+		const params = {
+			fields: 'id,title,mime',
+			include_deleted: '1',
+			include_conflicts: '1',
+		};
+		return await this.execPagedApiCommand_(
+			'GET',
+			'/resources',
+			params,
+			(item): ResourceData => ({
+				id: getStringProperty(item, 'id'),
+				title: getStringProperty(item, 'title'),
+				mimeType: getStringProperty(item, 'mime'),
+			}),
+		);
+	}
+
 	public async listNotes() {
 		const params = {
 			fields: 'id,parent_id,body,title,is_conflict,conflict_original_id,share_id,is_shared',
@@ -812,10 +997,14 @@ class Client implements ActionableClient {
 		return this.tracker_.itemById(itemId);
 	}
 
+	public itemExists(itemId: ItemId) {
+		return this.tracker_.itemExists(itemId);
+	}
+
 	public async checkState() {
 		logger.info('Check state', this.label);
 
-		type ItemSlice = { id: string };
+		type ItemSlice = { id: string; title: string };
 		const compare = (a: ItemSlice, b: ItemSlice) => {
 			if (a.id === b.id) return 0;
 			return a.id < b.id ? -1 : 1;
@@ -833,32 +1022,46 @@ class Client implements ActionableClient {
 			}
 		};
 
-		const assertSameIds = (actualSorted: ItemSlice[], expectedSorted: ItemSlice[], testLabel: string) => {
+		const idLogs = (ids: ItemId[], items: ItemSlice[]) => {
+			const itemTitle = (id: ItemId) => {
+				const itemTitle = items.find(item => item.id === id)?.title;
+				return itemTitle ? JSON.stringify(substrWithEllipsis(itemTitle, 0, 28)) : 'Unknown';
+			};
+
+			const output = [];
+			for (const id of ids) {
+				const log = this.globalActionTracker_.getActionLog(id);
+
+				output.push(`id: ${id} (${itemTitle(id)})`);
+				if (log.length > 0) {
+					output.push(
+						log
+							.map(item => `\t${item.source}: ${item.action}`)
+							.join('\n'),
+					);
+				} else {
+					output.push('\tNo history found');
+				}
+			}
+			return output.join('\n');
+		};
+
+		const assertSameIds = async (actualSorted: ItemSlice[], expectedSorted: ItemSlice[], assertionLabel: string) => {
 			const actualIds = actualSorted.map(i => i.id);
 			const expectedIds = expectedSorted.map(i => i.id);
 			const { missing, unexpected } = diffSortedStringArrays(actualIds, expectedIds);
 
-
 			if (missing.length || unexpected.length) {
-				const idLogs = (ids: string[]) => {
-					const output = [];
-					for (const id of ids) {
-						const log = this.globalActionTracker_.getActionLog(id);
-						output.push(`\nid:${id}`);
-						output.push(log.map(item => `\t${item.source}: ${item.action}`).join('\n'));
-					}
-					return output.join('\n');
-				};
-
-				throw new Error([
-					`IDs were different (${testLabel}):`,
-					missing.length && `- Expected ${JSON.stringify(missing)} to be present, but were missing.`,
-					unexpected.length && `- Present but should not have been: ${JSON.stringify(unexpected)}`,
-					'\n',
+				const message = [
+					`${assertionLabel}: IDs were different:`,
+					missing.length && `Expected ${JSON.stringify(missing)} to be present, but were missing.`,
+					unexpected.length && `Present but should not have been: ${JSON.stringify(unexpected)}`,
 					'Logs:',
-					idLogs(missing),
-					idLogs(unexpected),
-				].filter(line => !!line).join('\n'));
+					idLogs(missing, expectedSorted),
+					idLogs(unexpected, actualSorted),
+				].filter(line => !!line).join('\n');
+
+				throw new Error(message);
 			}
 		};
 
@@ -871,7 +1074,7 @@ class Client implements ActionableClient {
 
 			assertNoAdjacentEqualIds(notes, 'notes');
 			assertNoAdjacentEqualIds(expectedNotes, 'expectedNotes');
-			assertSameIds(notes, expectedNotes, 'should have the same note IDs');
+			await assertSameIds(notes, expectedNotes, 'Note IDs should match');
 			assert.deepEqual(notes, expectedNotes, 'should have the same notes as the expected state');
 		};
 
@@ -884,12 +1087,49 @@ class Client implements ActionableClient {
 
 			assertNoAdjacentEqualIds(folders, 'folders');
 			assertNoAdjacentEqualIds(expectedFolders, 'expectedFolders');
-			assertSameIds(folders, expectedFolders, 'should have the same folder IDs');
+			await assertSameIds(folders, expectedFolders, 'Folder IDs should match');
 			assert.deepEqual(folders, expectedFolders, 'should have the same folders as the expected state');
 		};
 
-		await checkNoteState();
-		await checkFolderState();
+		const checkResourceState = async () => {
+			const actualResources = [...await this.listResources()];
+			const actualResourceIds = new Set(actualResources.map(r => r.id));
+			const expectedResources = [...await this.tracker_.listResources()];
+
+			const missingResources = [];
+			for (const resource of expectedResources) {
+				if (!actualResourceIds.has(resource.id)) {
+					missingResources.push(resource.id);
+				}
+			}
+
+			if (missingResources.length > 0) {
+				const log = idLogs(missingResources, expectedResources);
+
+				throw new Error(`Missing resource(s): All expected resources should exist on the client. Resource(s) with ID(s) ${JSON.stringify(missingResources)} were not found (total resource count: ${actualResourceIds.size}).\nResource action history:\n${log}`);
+			}
+		};
+
+		const errors: Error[] = [];
+		const runCheck = async (check: ()=> Promise<void>) => {
+			try {
+				await check();
+			} catch (error) {
+				errors.push(error);
+			}
+		};
+
+		await runCheck(checkResourceState);
+		await runCheck(checkNoteState);
+		await runCheck(checkFolderState);
+
+		if (errors.length) {
+			const errorList = errors
+				.map((error, index) => `Error ${index + 1} of ${errors.length}: ${error}`)
+				.map(message => hangingIndent(message))
+				.join('\n');
+			throw new Error(`Incorrect state in client: ${this.clientLabel_}:\n${errorList}`);
+		}
 	}
 }
 

packages/tools/fuzzer/model/FolderRecord.ts

@@ -19,8 +19,8 @@ const validateId = (id: string) => {
 };
 
 export default class FolderRecord implements FolderData {
-	public readonly parentId: string;
-	public readonly id: string;
+	public readonly parentId: ItemId;
+	public readonly id: ItemId;
 	public readonly title: string;
 	public readonly ownedByEmail: string;
 	public readonly childIds: ItemId[];

packages/tools/fuzzer/model/ResourceRecord.ts

@@ -0,0 +1,49 @@
+import { ItemId, ResourceData } from '../types';
+
+interface InitializationOptions extends ResourceData {
+	referencedBy: ItemId[];
+}
+
+export default class ResourceRecord implements ResourceData {
+	public readonly parentId: undefined;
+	public readonly id: ItemId;
+	public readonly title: string;
+	public readonly mimeType: string;
+	public readonly referencedBy: readonly ItemId[] = [];
+
+	public constructor(options: InitializationOptions) {
+		this.id = options.id;
+		this.title = options.title;
+		this.mimeType = options.mimeType;
+		this.referencedBy = [...options.referencedBy];
+	}
+
+	public get referenceCount() {
+		return this.referencedBy.length;
+	}
+
+	public withReference(noteId: ItemId) {
+		if (this.referencedBy.includes(noteId)) {
+			return this;
+		}
+		return new ResourceRecord({
+			id: this.id,
+			title: this.title,
+			mimeType: this.mimeType,
+			referencedBy: [...this.referencedBy, noteId],
+		});
+	}
+
+	public withoutReference(noteId: ItemId) {
+		if (this.referencedBy.includes(noteId)) {
+			return this;
+		}
+
+		return new ResourceRecord({
+			id: this.id,
+			title: this.title,
+			mimeType: this.mimeType,
+			referencedBy: this.referencedBy.filter(ref => ref !== noteId),
+		});
+	}
+}

packages/tools/fuzzer/sync-fuzzer.ts

@@ -13,6 +13,7 @@ import { packagesDir } from './constants';
 import ActionRunner, { ActionSpec } from './ActionRunner';
 import randomString from './utils/randomString';
 import { readFile } from 'fs/promises';
+import randomId from './utils/randomId';
 const { shimInit } = require('@joplin/lib/shim-init-node');
 
 const globalLogger = new Logger();
@@ -55,9 +56,10 @@ interface Options {
 
 const createContext = (options: Options, server: Server, profilesDirectory: string) => {
 	const random = new SeededRandom(options.seed);
-	// Use a separate random number generator for strings. This prevents
+	// Use a separate random number generator for strings and IDs. This prevents
 	// the random strings setting from affecting the other output.
 	const stringRandom = new SeededRandom(random.next());
+	const idRandom = new SeededRandom(random.next());
 
 	if (options.isJoplinCloud) {
 		logger.info('Sync target: Joplin Cloud');
@@ -71,6 +73,7 @@ const createContext = (options: Options, server: Server, profilesDirectory: stri
 			return (_targetLength: number) => `Placeholder (x${stringCount++})`;
 		}
 	})();
+	const randomIdGenerator = randomId((min, max) => idRandom.nextInRange(min, max));
 
 	const fuzzContext: FuzzContext = {
 		serverUrl: server.url,
@@ -82,6 +85,7 @@ const createContext = (options: Options, server: Server, profilesDirectory: stri
 		randInt: (a, b) => random.nextInRange(a, b),
 		randomFrom: (data) => data[random.nextInRange(0, data.length)],
 		randomString: randomStringGenerator,
+		randomId: randomIdGenerator,
 		keepAccounts: options.keepAccountsOnClose,
 	};
 	return fuzzContext;

packages/tools/fuzzer/types.ts

@@ -1,5 +1,6 @@
 import type Client from './Client';
 import type FolderRecord from './model/FolderRecord';
+import ResourceRecord from './model/ResourceRecord';
 
 export type Json = string|number|Json[]|{ [key: string]: Json };
 
@@ -25,12 +26,26 @@ export interface DetailedFolderData extends FolderData {
 	isShared: boolean;
 }
 
-export type TreeItem = NoteData|FolderRecord;
+export interface ResourceData {
+	id: ItemId;
+	title: string;
+	mimeType: string;
+}
+
+export type TreeItem = NoteData|FolderRecord|ResourceRecord;
 
 export const isFolder = (item: TreeItem): item is FolderRecord => {
 	return 'childIds' in item;
 };
 
+export const isResource = (item: TreeItem): item is ResourceRecord => {
+	return 'mimeType' in item;
+};
+
+export const isNote = (item: TreeItem): item is NoteData => {
+	return !isFolder(item) && !isResource(item);
+};
+
 // Typescript type assertions require type definitions on the left for arrow functions.
 // See https://github.com/microsoft/TypeScript/issues/53450.
 export const assertIsFolder: (item: TreeItem)=> asserts item is FolderRecord = item => {
@@ -57,6 +72,7 @@ export interface FuzzContext {
 	execApi: (method: HttpMethod, route: string, debugAction: Json)=> Promise<Json>;
 	randInt: (low: number, high: number)=> number;
 	randomString: (targetLength: number)=> string;
+	randomId: ()=> string;
 	randomFrom: <T> (data: T[])=> T;
 }
 
@@ -82,6 +98,8 @@ export interface ActionableClient {
 	deleteNote(id: ItemId): Promise<void>;
 	createNote(data: NoteData): Promise<void>;
 	updateNote(data: NoteData): Promise<void>;
+	attachResource(note: NoteData, resource: ResourceData): Promise<NoteData>;
+	createResource(resource: ResourceData): Promise<void>;
 	moveItem(itemId: ItemId, newParentId: ItemId): Promise<void>;
 	publishNote(id: ItemId): Promise<void>;
 	unpublishNote(id: ItemId): Promise<void>;
@@ -89,10 +107,12 @@ export interface ActionableClient {
 
 	listNotes(): Promise<NoteData[]>;
 	listFolders(): Promise<DetailedFolderData[]>;
+	listResources(): Promise<ResourceData[]>;
 	allFolderDescendants(parentId: ItemId): Promise<ItemId[]>;
 	randomFolder(options: RandomFolderOptions): Promise<FolderRecord>;
 	randomNote(options: RandomNoteOptions): Promise<NoteData>;
 	itemById(id: ItemId): TreeItem;
+	itemExists(id: ItemId): boolean;
 }
 
 export interface UserData {

packages/tools/fuzzer/utils/extractResourceIds.ts

@@ -0,0 +1,7 @@
+import { extractResourceUrls } from '@joplin/lib/urlUtils';
+
+const extractResourceIds = (text: string) => {
+	return extractResourceUrls(text).map(item => item.itemId);
+};
+
+export default extractResourceIds;

packages/tools/fuzzer/utils/getProperty.ts

@@ -5,7 +5,9 @@ const getProperty = (object: unknown, propertyName: string) => {
 	}
 
 	if (!(propertyName in object)) {
-		throw new Error(`No such property ${JSON.stringify(propertyName)} in object`);
+		throw new Error(
+			`No such property ${JSON.stringify(propertyName)} in object. Available keys: (${JSON.stringify(Object.keys(object))})`,
+		);
 	}
 
 	return object[propertyName as keyof object];

packages/tools/fuzzer/utils/hangingIndent.ts

@@ -0,0 +1,7 @@
+
+// Hanging indent: Indents all lines after the first
+const hangingIndent = (text: string, indentation = '   ') => {
+	return text.replace(/\n/g, `\n${indentation}`);
+};
+
+export default hangingIndent;

packages/tools/fuzzer/utils/randomId.test.ts

@@ -0,0 +1,12 @@
+import randomId from './randomId';
+
+describe('randomId', () => {
+	test('should generate a 32-character alphanumeric ID', () => {
+		expect(
+			randomId((_low, high) => high - 1)(),
+		).toBe('ffffffffffffffffffffffffffffffff');
+		expect(
+			randomId((low, _high) => low)(),
+		).toBe('00000000000000000000000000000000');
+	});
+});

packages/tools/fuzzer/utils/randomId.ts

@@ -0,0 +1,16 @@
+
+type OnNextRandom = (lowInclusive: number, highExclusive: number)=> number;
+
+const randomId = (nextRandomInteger: OnNextRandom)=> () => {
+	const bytes = [];
+	for (let i = 0; i < 16; i++) {
+		bytes.push(nextRandomInteger(0, 256));
+	}
+
+	return Buffer.from(bytes)
+		.toString('hex')
+		.toLowerCase()
+		.padStart(32, '0');
+};
+
+export default randomId;

packages/tools/fuzzer/utils/retryWithCount.ts

@@ -3,10 +3,15 @@ import { msleep } from '@joplin/utils/time';
 
 const logger = Logger.create('retryWithCount');
 
+interface FailureEvent {
+	error: Error;
+	willRetry: boolean;
+}
+
 interface Options {
 	count: number;
 	delayOnFailure?: (retryCount: number)=> number;
-	onFail: (error: Error)=> void|Promise<void>;
+	onFail: (event: FailureEvent)=> void|Promise<void>;
 }
 
 const retryWithCount = async (task: ()=> Promise<void>, { count, delayOnFailure, onFail }: Options) => {
@@ -15,10 +20,11 @@ const retryWithCount = async (task: ()=> Promise<void>, { count, delayOnFailure,
 		try {
 			return await task();
 		} catch (error) {
-			await onFail(error);
 			lastError = error;
 
 			const willRetry = retry + 1 < count;
+			await onFail({ error, willRetry });
+
 			const delay = willRetry && delayOnFailure ? delayOnFailure(retry + 1) : 0;
 			if (delay) {
 				logger.info(`Retrying after ${delay}ms...`);