Chore: Tests: Improve Playwright test reliability (#10981)

packages/app-desktop/integration-tests/markdownEditor.spec.ts

@@ -13,11 +13,16 @@ test.describe('markdownEditor', () => {
 		await mainScreen.importHtmlDirectory(electronApp, join(__dirname, 'resources', 'html-import'));
 		const importedFolder = mainScreen.sidebar.container.getByText('html-import');
 		await importedFolder.waitFor();
+
+		// Retry -- focusing the imported-folder may fail in some cases
+		await expect(async () => {
 			await importedFolder.click();
 
 			await mainScreen.noteList.focusContent(electronApp);
+
 			const importedHtmlFileItem = mainScreen.noteList.getNoteItemByTitle('test-html-file-with-image');
-		await importedHtmlFileItem.click();
+			await importedHtmlFileItem.click({ timeout: 300 });
+		}).toPass();
 
 		const viewerFrame = mainScreen.noteEditor.getNoteViewerIframe();
 		// Should render headers

packages/app-desktop/integration-tests/models/GoToAnything.ts

@@ -14,10 +14,7 @@ export default class GoToAnything {
 
 	public async open(electronApp: ElectronApplication) {
 		await this.mainScreen.waitFor();
-
-		if (!await activateMainMenuItem(electronApp, 'Goto Anything...')) {
-			throw new Error('Menu item for opening Goto Anything not found');
-		}
+		await activateMainMenuItem(electronApp, 'Goto Anything...');
 
 		return this.waitFor();
 	}

packages/app-desktop/integration-tests/models/MainScreen.ts

@@ -46,12 +46,7 @@ export default class MainScreen {
 
 	public async openSettings(electronApp: ElectronApplication) {
 		// Check both labels so this works on MacOS
-		const openedWithPreferences = await activateMainMenuItem(electronApp, 'Preferences...');
-		const openedWithOptions = await activateMainMenuItem(electronApp, 'Options');
-
-		if (!openedWithOptions && !openedWithPreferences) {
-			throw new Error('Unable to find settings menu item in application menus.');
-		}
+		await activateMainMenuItem(electronApp, /^(Preferences\.\.\.|Options)$/);
 	}
 
 	public async search(text: string) {
@@ -61,10 +56,6 @@ export default class MainScreen {
 
 	public async importHtmlDirectory(electronApp: ElectronApplication, path: string) {
 		await setFilePickerResponse(electronApp, [path]);
-		const startedImport = await activateMainMenuItem(electronApp, 'HTML - HTML document (Directory)', 'Import');
-
-		if (!startedImport) {
-			throw new Error('Unable to find HTML directory import menu item.');
-		}
+		await activateMainMenuItem(electronApp, 'HTML - HTML document (Directory)', 'Import');
 	}
 }

packages/app-desktop/integration-tests/models/NoteList.ts

@@ -3,9 +3,11 @@ import { ElectronApplication, Locator, Page, expect } from '@playwright/test';
 
 export default class NoteList {
 	public readonly container: Locator;
+	public readonly sortOrderButton: Locator;
 
 	public constructor(page: Page) {
 		this.container = page.locator('.rli-noteList');
+		this.sortOrderButton = this.container.getByRole('button', { name: 'Toggle sort order' });
 	}
 
 	public waitFor() {
@@ -13,14 +15,12 @@ export default class NoteList {
 	}
 
 	private async sortBy(electronApp: ElectronApplication, sortMethod: string) {
-		const success = await activateMainMenuItem(electronApp, sortMethod, 'Sort notes by');
-		if (!success) {
-			throw new Error(`Unable to find sorting menu item: ${sortMethod}`);
-		}
+		await activateMainMenuItem(electronApp, sortMethod, 'Sort notes by');
 	}
 
 	public async sortByTitle(electronApp: ElectronApplication) {
-		return this.sortBy(electronApp, 'Title');
+		await this.sortBy(electronApp, 'Title');
+		await expect(this.sortOrderButton).toHaveAttribute('title', /Toggle sort order field:[\n ]*title ->/);
 	}
 
 	public async focusContent(electronApp: ElectronApplication) {

packages/app-desktop/integration-tests/models/Sidebar.ts

@@ -23,10 +23,7 @@ export default class Sidebar {
 	}
 
 	private async sortBy(electronApp: ElectronApplication, option: string) {
-		const success = await activateMainMenuItem(electronApp, option, 'Sort notebooks by');
-		if (!success) {
-			throw new Error(`Failed to find menu item: ${option}`);
-		}
+		await activateMainMenuItem(electronApp, option, 'Sort notebooks by');
 	}
 
 	public async sortByDate(electronApp: ElectronApplication) {

packages/app-desktop/integration-tests/noteList.spec.ts

@@ -91,7 +91,14 @@ test.describe('noteList', () => {
 		await noteList.focusContent(electronApp);
 		// The most recently-created note should be visible
 		const note4Item = noteList.getNoteItemByTitle('note_4');
+		const note3Item = noteList.getNoteItemByTitle('note_3');
+		const note2Item = noteList.getNoteItemByTitle('note_2');
+		const note1Item = noteList.getNoteItemByTitle('note_1');
 		await expect(note4Item).toBeVisible();
+		await expect(note3Item).toBeVisible();
+		await expect(note2Item).toBeVisible();
+		await expect(note1Item).toBeVisible();
+
 		await noteList.expectNoteToBeSelected('note_4');
 
 		await noteList.container.press('ArrowUp');

packages/app-desktop/integration-tests/settings.spec.ts

@@ -8,7 +8,7 @@ test.describe('settings', () => {
 		await mainScreen.waitFor();
 
 		// Sort order buttons should be visible by default
-		const sortOrderLocator = mainScreen.noteList.container.getByRole('button', { name: 'Toggle sort order' });
+		const sortOrderLocator = mainScreen.noteList.sortOrderButton;
 		await expect(sortOrderLocator).toBeVisible();
 
 		await mainScreen.openSettings(electronApp);

packages/app-desktop/integration-tests/simpleBackup.spec.ts

@@ -1,4 +1,4 @@
-import { test, expect } from './util/test';
+import { test } from './util/test';
 import MainScreen from './models/MainScreen';
 import SettingsScreen from './models/SettingsScreen';
 import activateMainMenuItem from './util/activateMainMenuItem';
@@ -28,7 +28,7 @@ test.describe('simpleBackup', () => {
 		await mainScreen.waitFor();
 
 		// Backups should work
-		expect(await activateMainMenuItem(electronApp, 'Create backup')).toBe(true);
+		await activateMainMenuItem(electronApp, 'Create backup');
 
 		const successDialog = mainWindow.locator('iframe[id$=backup-backupDialog]');
 		await successDialog.waitFor();

packages/app-desktop/integration-tests/util/activateMainMenuItem.ts

@@ -1,5 +1,5 @@
 
-import type { ElectronApplication } from '@playwright/test';
+import { expect, type ElectronApplication } from '@playwright/test';
 import type { MenuItem } from 'electron';
 
 
@@ -7,16 +7,23 @@ import type { MenuItem } from 'electron';
 // https://github.com/spaceagetv/electron-playwright-helpers/blob/main/src/menu_helpers.ts
 
 // If given, `parentMenuLabel` should be the label of the menu containing the target item.
-const activateMainMenuItem = (
+const activateMainMenuItem = async (
 	electronApp: ElectronApplication,
-	targetItemLabel: string,
+	targetItemLabel: string|RegExp,
 	parentMenuLabel?: string,
 ) => {
+	await expect.poll(() => {
 		return electronApp.evaluate(async ({ Menu }, [targetItemLabel, parentMenuLabel]) => {
 			const activateItemInSubmenu = (submenu: MenuItem[], parentLabel: string) => {
 				for (const item of submenu) {
 					const matchesParent = !parentMenuLabel || parentLabel === parentMenuLabel;
-				if (item.label === targetItemLabel && matchesParent && item.visible) {
+					const matchesLabel = typeof targetItemLabel === 'string' ? (
+						targetItemLabel === item.label
+					) : (
+						item.label.match(targetItemLabel)
+					);
+
+					if (matchesLabel && matchesParent && item.visible) {
 						// Found!
 						item.click();
 						return true;
@@ -36,6 +43,9 @@ const activateMainMenuItem = (
 			const appMenu = Menu.getApplicationMenu();
 			return activateItemInSubmenu(appMenu.items, '');
 		}, [targetItemLabel, parentMenuLabel]);
+	}, {
+		message: `should find and activate menu item with label ${JSON.stringify(targetItemLabel)}`,
+	}).toBe(true);
 };
 
 export default activateMainMenuItem;

packages/app-desktop/integration-tests/util/test.ts

@@ -56,6 +56,13 @@ export const test = base.extend<JoplinFixtures>({
 
 	mainWindow: async ({ electronApp }, use) => {
 		const mainWindow = await firstNonDevToolsWindow(electronApp);
+
+		// Setting the viewport size helps keep test environments consistent.
+		await mainWindow.setViewportSize({
+			width: 1200,
+			height: 800,
+		});
+
 		await use(mainWindow);
 	},
 });