Chore: Desktop: Set up integration testing with Playwright (#9043)

.eslintignore

@@ -38,6 +38,10 @@ packages/app-clipper/popup/config/webpack.config.js
 packages/app-clipper/popup/node_modules
 packages/app-clipper/popup/scripts/build.js
 packages/app-desktop/build/
+packages/app-desktop/test-results/
+packages/app-desktop/playwright-report/
+packages/app-desktop/playwright/.cache/
+packages/app-desktop/integration-tests/test-profile/
 packages/app-desktop/dist
 packages/app-desktop/gui/NoteEditor/NoteBody/TinyMCE/plugins/lists.js
 packages/app-desktop/gui/NoteEditor/NoteBody/TinyMCE/supportedLocales.js
@@ -371,6 +375,13 @@ packages/app-desktop/gui/style/StyledTextInput.js
 packages/app-desktop/gui/utils/NoteListUtils.js
 packages/app-desktop/gui/utils/convertToScreenCoordinates.js
 packages/app-desktop/gui/utils/loadScript.js
+packages/app-desktop/integration-tests/main.spec.js
+packages/app-desktop/integration-tests/models/MainScreen.js
+packages/app-desktop/integration-tests/models/NoteEditorScreen.js
+packages/app-desktop/integration-tests/models/SettingsScreen.js
+packages/app-desktop/integration-tests/util/activateMainMenuItem.js
+packages/app-desktop/integration-tests/util/test.js
+packages/app-desktop/playwright.config.js
 packages/app-desktop/plugins/GotoAnything.js
 packages/app-desktop/services/bridge.js
 packages/app-desktop/services/commands/stateToWhenClauseContext.js

.github/workflows/github-actions-main.yml

@@ -55,6 +55,9 @@ jobs:
           sudo apt-get install -y libsecret-1-dev
           sudo apt-get install -y translate-toolkit
           sudo apt-get install -y rsync
+          # Provides a virtual display on Linux. Used for Playwright integration
+          # testing.
+          sudo apt-get install -y xvfb
 
       - name: Install Docker Engine
         # if: runner.os == 'Linux' && startsWith(github.ref, 'refs/tags/server-v')

.gitignore

@@ -357,6 +357,13 @@ packages/app-desktop/gui/style/StyledTextInput.js
 packages/app-desktop/gui/utils/NoteListUtils.js
 packages/app-desktop/gui/utils/convertToScreenCoordinates.js
 packages/app-desktop/gui/utils/loadScript.js
+packages/app-desktop/integration-tests/main.spec.js
+packages/app-desktop/integration-tests/models/MainScreen.js
+packages/app-desktop/integration-tests/models/NoteEditorScreen.js
+packages/app-desktop/integration-tests/models/SettingsScreen.js
+packages/app-desktop/integration-tests/util/activateMainMenuItem.js
+packages/app-desktop/integration-tests/util/test.js
+packages/app-desktop/playwright.config.js
 packages/app-desktop/plugins/GotoAnything.js
 packages/app-desktop/services/bridge.js
 packages/app-desktop/services/commands/stateToWhenClauseContext.js

packages/app-desktop/.gitignore

@@ -14,3 +14,7 @@ style.min.css
 build/lib/
 vendor/*
 !vendor/loadEmojiLib.js
+test-results/
+playwright-report/
+playwright/.cache/
+integration-tests/test-profile/

packages/app-desktop/integration-tests/README.md

@@ -0,0 +1,17 @@
+# Integration tests
+
+The integration tests in this directory can be run with `yarn playwright test`.
+
+- Tests use a `test-profile` directory that should be re-created before every test.
+- Only one Electron application should be instantiated per test file.
+- Files in the `models/` directory follow [the page object model](https://playwright.dev/docs/pom).
+
+# References
+
+The following sources are helpful for designing and implementing Electron integration tests
+with Playwright:
+- [A setup guide from an organisation that uses Playwright](https://dev.to/kubeshop/testing-electron-apps-with-playwright-3f89)
+  and [that organisation's test suite](https://github.com/kubeshop/monokle/blob/main/tests/base.test.ts).
+- [The Playwright ElectronApp docs](https://playwright.dev/docs/api/class-electronapplication)
+- [Electron Playwright example repository](https://github.com/spaceagetv/electron-playwright-example)
+- [Playwright best practices](https://playwright.dev/docs/best-practices)

packages/app-desktop/integration-tests/main.spec.ts

@@ -0,0 +1,79 @@
+import { test, expect } from './util/test';
+import MainScreen from './models/MainScreen';
+import activateMainMenuItem from './util/activateMainMenuItem';
+import SettingsScreen from './models/SettingsScreen';
+
+
+test.describe('main', () => {
+	test('app should launch', async ({ mainWindow }) => {
+		// A window should open with the correct title
+		expect(await mainWindow.title()).toMatch(/^Joplin/);
+
+		const mainPage = new MainScreen(mainWindow);
+		await mainPage.waitFor();
+	});
+
+	test('should be able to create and edit a new note', async ({ mainWindow }) => {
+		const mainScreen = new MainScreen(mainWindow);
+		await mainScreen.newNoteButton.click();
+
+		const editor = mainScreen.noteEditor;
+		await editor.waitFor();
+
+		// Wait for the title input to have the correct placeholder
+		await mainWindow.locator('input[placeholder^="Creating new note"]').waitFor();
+
+		// Fill the title
+		await editor.noteTitleInput.click();
+		await editor.noteTitleInput.fill('Test note');
+
+		// Note list should contain the new note
+		await expect(mainScreen.noteListContainer.getByText('Test note')).toBeVisible();
+
+		// Focus the editor
+		await editor.codeMirrorEditor.click();
+
+		// Type some text
+		await mainWindow.keyboard.type('# Test note!');
+		await mainWindow.keyboard.press('Enter');
+		await mainWindow.keyboard.press('Enter');
+		await mainWindow.keyboard.type('New note content!');
+
+		// Should render
+		const viewerFrame = editor.getNoteViewerIframe();
+		await expect(viewerFrame.locator('h1')).toHaveText('Test note!');
+	});
+
+	test('should be possible to remove sort order buttons in settings', async ({ electronApp, mainWindow }) => {
+		const mainScreen = new MainScreen(mainWindow);
+		await mainScreen.waitFor();
+
+		// Sort order buttons should be visible by default
+		await expect(mainScreen.noteListContainer.locator('[title^="Toggle sort order"]')).toBeVisible();
+
+		// Open settings (check both labels so that this works on MacOS)
+		expect(
+			await activateMainMenuItem(electronApp, 'Preferences...') || await activateMainMenuItem(electronApp, 'Options'),
+		).toBe(true);
+
+		// Should be on the settings screen
+		const settingsScreen = new SettingsScreen(mainWindow);
+		await settingsScreen.waitFor();
+
+		// Open the appearance tab
+		await settingsScreen.appearanceTabButton.click();
+
+		// Find the sort order visible checkbox
+		const sortOrderVisibleCheckbox = mainWindow.getByLabel(/^Show sort order/);
+
+		await expect(sortOrderVisibleCheckbox).toBeChecked();
+		await sortOrderVisibleCheckbox.click();
+		await expect(sortOrderVisibleCheckbox).not.toBeChecked();
+
+		// Save settings & close
+		await settingsScreen.okayButton.click();
+		await mainScreen.waitFor();
+
+		await expect(mainScreen.noteListContainer.locator('[title^="Toggle sort order"]')).not.toBeVisible();
+	});
+});

packages/app-desktop/integration-tests/models/MainScreen.ts

@@ -0,0 +1,20 @@
+import { Page, Locator } from '@playwright/test';
+import NoteEditorScreen from './NoteEditorScreen';
+
+export default class MainScreen {
+	public readonly newNoteButton: Locator;
+	public readonly noteListContainer: Locator;
+	public readonly noteEditor: NoteEditorScreen;
+
+	public constructor(page: Page) {
+		this.newNoteButton = page.locator('.new-note-button');
+		this.noteListContainer = page.locator('.rli-noteList');
+		this.noteEditor = new NoteEditorScreen(page);
+	}
+
+	public async waitFor() {
+		await this.newNoteButton.waitFor();
+		await this.noteEditor.waitFor();
+		await this.noteListContainer.waitFor();
+	}
+}

packages/app-desktop/integration-tests/models/NoteEditorScreen.ts

@@ -0,0 +1,26 @@
+
+import { Locator, Page } from '@playwright/test';
+
+export default class NoteEditorPage {
+	public readonly codeMirrorEditor: Locator;
+	public readonly noteTitleInput: Locator;
+	private readonly containerLocator: Locator;
+
+	public constructor(private readonly page: Page) {
+		this.containerLocator = page.locator('.rli-editor');
+		this.codeMirrorEditor = this.containerLocator.locator('.codeMirrorEditor');
+		this.noteTitleInput = this.containerLocator.locator('.title-input');
+	}
+
+	public getNoteViewerIframe() {
+		// The note viewer can change content when the note re-renders. As such,
+		// a new locator needs to be created after re-renders (and this can't be a
+		// static property).
+		return this.page.frame({ url: /.*note-viewer[/\\]index.html.*/ });
+	}
+
+	public async waitFor() {
+		await this.codeMirrorEditor.waitFor();
+		await this.noteTitleInput.waitFor();
+	}
+}

packages/app-desktop/integration-tests/models/SettingsScreen.ts

@@ -0,0 +1,17 @@
+
+import { Page, Locator } from '@playwright/test';
+
+export default class SettingsScreen {
+	public readonly okayButton: Locator;
+	public readonly appearanceTabButton: Locator;
+
+	public constructor(page: Page) {
+		this.okayButton = page.locator('button', { hasText: 'OK' });
+		this.appearanceTabButton = page.getByText('Appearance');
+	}
+
+	public async waitFor() {
+		await this.okayButton.waitFor();
+		await this.appearanceTabButton.waitFor();
+	}
+}

packages/app-desktop/integration-tests/run-ci.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+echo "Running desktop integration tests..."
+
+export CI=true
+
+if test "$RUNNER_OS" = "Linux" ; then
+	# The Ubuntu Github CI doesn't have a display server.
+	# Start a virtual one with xvfb-run. 
+	xvfb-run -- yarn run playwright test
+else
+	yarn run playwright test
+fi

packages/app-desktop/integration-tests/util/activateMainMenuItem.ts

@@ -0,0 +1,36 @@
+
+import type { ElectronApplication } from '@playwright/test';
+import type { MenuItem } from 'electron';
+
+
+// Roughly based on
+// https://github.com/spaceagetv/electron-playwright-helpers/blob/main/src/menu_helpers.ts
+
+// `menuItemPath` should be a list of menu labels (e.g. [["&JoplinMainMenu", "&File"], "Synchronise"]).
+const activateMainMenuItem = (electronApp: ElectronApplication, menuItemLabel: string) => {
+	return electronApp.evaluate(async ({ Menu }, menuItemLabel) => {
+		const activateItemInSubmenu = (submenu: MenuItem[]) => {
+			for (const item of submenu) {
+				if (item.label === menuItemLabel && item.visible) {
+					// Found!
+					item.click();
+					return true;
+				} else if (item.submenu) {
+					const foundItem = activateItemInSubmenu(item.submenu.items);
+
+					if (foundItem) {
+						return true;
+					}
+				}
+			}
+
+			// No item found
+			return false;
+		};
+
+		const appMenu = Menu.getApplicationMenu();
+		return activateItemInSubmenu(appMenu.items);
+	}, menuItemLabel);
+};
+
+export default activateMainMenuItem;

packages/app-desktop/integration-tests/util/test.ts

@@ -0,0 +1,45 @@
+import { resolve, join, dirname } from 'path';
+import { remove, mkdirp } from 'fs-extra';
+import { _electron as electron, Page, ElectronApplication, test as base } from '@playwright/test';
+import uuid from '@joplin/lib/uuid';
+
+
+
+type JoplinFixtures = {
+	electronApp: ElectronApplication;
+	mainWindow: Page;
+};
+
+// A custom fixture that loads an electron app. See
+// https://playwright.dev/docs/test-fixtures
+
+export const test = base.extend<JoplinFixtures>({
+	// Playwright fails if we don't use the object destructuring
+	// pattern in the first argument.
+	//
+	// See https://github.com/microsoft/playwright/issues/8798
+	//
+	// eslint-disable-next-line no-empty-pattern
+	electronApp: async ({ }, use) => {
+		const profilePath = resolve(join(dirname(__dirname), 'test-profile'));
+		const profileSubdir = join(profilePath, uuid.createNano());
+		await mkdirp(profileSubdir);
+
+		const startupArgs = ['main.js', '--env', 'dev', '--profile', profileSubdir];
+		const electronApp = await electron.launch({ args: startupArgs });
+
+		await use(electronApp);
+
+		await electronApp.firstWindow();
+		await electronApp.close();
+		await remove(profileSubdir);
+	},
+
+	mainWindow: async ({ electronApp }, use) => {
+		const window = await electronApp.firstWindow();
+		await use(window);
+	},
+});
+
+
+export { expect } from '@playwright/test';

packages/app-desktop/package.json

@@ -14,7 +14,8 @@
     "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json",
     "start": "gulp build && electron . --env dev --log-level debug --open-dev-tools",
     "test": "jest",
-    "test-ci": "yarn test",
+    "test-ui": "playwright test",
+    "test-ci": "yarn test && sh ./integration-tests/run-ci.sh",
     "renameReleaseAssets": "node tools/renameReleaseAssets.js"
   },
   "repository": {
@@ -116,6 +117,7 @@
   "devDependencies": {
     "@electron/rebuild": "3.3.0",
     "@joplin/tools": "~2.13",
+    "@playwright/test": "1.38.1",
     "@testing-library/react-hooks": "8.0.1",
     "@types/jest": "29.5.4",
     "@types/node": "18.17.18",

packages/app-desktop/playwright.config.ts

@@ -0,0 +1,34 @@
+import { defineConfig } from '@playwright/test';
+
+// See https://playwright.dev/docs/test-configuration.
+export default defineConfig({
+	testDir: './integration-tests',
+
+	// Only match .ts files (no compiled .js files)
+	testMatch: '*.spec.ts',
+
+	// Allow running tests in parallel (note: each Joplin instance
+	// is given its own profile directory).
+	fullyParallel: true,
+
+	// Fail the build on CI if you accidentally left test.only in the source code.
+	forbidOnly: !!process.env.CI,
+
+	// Retry on CI only
+	retries: process.env.CI ? 2 : 0,
+
+	// Opt out of parallel tests on CI
+	workers: process.env.CI ? 1 : undefined,
+
+	// Reporter to use. See https://playwright.dev/docs/test-reporters
+	reporter: process.env.CI ? 'line' : 'html',
+
+	// Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions.
+	use: {
+		// Base URL to use in actions like `await page.goto('/')`.
+		// baseURL: 'http://127.0.0.1:3000',
+
+		// Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer
+		trace: 'on-first-retry',
+	},
+});

yarn.lock

@@ -4661,6 +4661,7 @@ __metadata:
     "@joplin/renderer": ~2.13
     "@joplin/tools": ~2.13
     "@joplin/utils": ~2.13
+    "@playwright/test": 1.38.1
     "@testing-library/react-hooks": 8.0.1
     "@types/jest": 29.5.4
     "@types/mustache": 4.2.2
@@ -6793,6 +6794,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@playwright/test@npm:1.38.1":
+  version: 1.38.1
+  resolution: "@playwright/test@npm:1.38.1"
+  dependencies:
+    playwright: 1.38.1
+  bin:
+    playwright: cli.js
+  checksum: c5ec0b23261fe1ef163b6234f69263bc10e7e5a3fb676c7773ffc70b87459a7ab225f57c03b9de649475771638a04c2e00d9b2739304a4dcf5d3edf20a7a4a82
+  languageName: node
+  linkType: hard
+
 "@popperjs/core@npm:^2.4.0":
   version: 2.11.0
   resolution: "@popperjs/core@npm:2.11.0"
@@ -17745,6 +17757,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"fsevents@npm:2.3.2, fsevents@npm:^2.1.2, fsevents@npm:^2.3.2, fsevents@npm:~2.3.2":
+  version: 2.3.2
+  resolution: "fsevents@npm:2.3.2"
+  dependencies:
+    node-gyp: latest
+  checksum: 97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f
+  conditions: os=darwin
+  languageName: node
+  linkType: hard
+
 "fsevents@npm:^1.2.7":
   version: 1.2.13
   resolution: "fsevents@npm:1.2.13"
@@ -17756,12 +17778,11 @@ __metadata:
   languageName: node
   linkType: hard
 
-"fsevents@npm:^2.1.2, fsevents@npm:^2.3.2, fsevents@npm:~2.3.2":
+"fsevents@patch:fsevents@2.3.2#~builtin<compat/fsevents>, fsevents@patch:fsevents@^2.1.2#~builtin<compat/fsevents>, fsevents@patch:fsevents@^2.3.2#~builtin<compat/fsevents>, fsevents@patch:fsevents@~2.3.2#~builtin<compat/fsevents>":
   version: 2.3.2
-  resolution: "fsevents@npm:2.3.2"
+  resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin<compat/fsevents>::version=2.3.2&hash=df0bf1"
   dependencies:
     node-gyp: latest
-  checksum: 97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f
   conditions: os=darwin
   languageName: node
   linkType: hard
@@ -17776,15 +17797,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"fsevents@patch:fsevents@^2.1.2#~builtin<compat/fsevents>, fsevents@patch:fsevents@^2.3.2#~builtin<compat/fsevents>, fsevents@patch:fsevents@~2.3.2#~builtin<compat/fsevents>":
-  version: 2.3.2
-  resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin<compat/fsevents>::version=2.3.2&hash=df0bf1"
-  dependencies:
-    node-gyp: latest
-  conditions: os=darwin
-  languageName: node
-  linkType: hard
-
 "fullstore@npm:^1.0.0":
   version: 1.1.0
   resolution: "fullstore@npm:1.1.0"
@@ -28156,6 +28168,30 @@ __metadata:
   languageName: node
   linkType: hard
 
+"playwright-core@npm:1.38.1":
+  version: 1.38.1
+  resolution: "playwright-core@npm:1.38.1"
+  bin:
+    playwright-core: cli.js
+  checksum: 66e83fe040f309b13ad94ba39dea40ac207bfcbbc22de13141af88dbdedd64e1c4e3ce1d0cb070d4efd8050d7e579953ec3681dd8a0acf2c1cc738d9c50e545e
+  languageName: node
+  linkType: hard
+
+"playwright@npm:1.38.1":
+  version: 1.38.1
+  resolution: "playwright@npm:1.38.1"
+  dependencies:
+    fsevents: 2.3.2
+    playwright-core: 1.38.1
+  dependenciesMeta:
+    fsevents:
+      optional: true
+  bin:
+    playwright: cli.js
+  checksum: 4e01d4ee52d9ccf75a80d8492829106802590721d56bff7c5957ff1f21eb3c328ee5bc3c1784a59c4b515df1b98d08ef92e4a35a807f454cd00dc481d30fadc2
+  languageName: node
+  linkType: hard
+
 "please-upgrade-node@npm:^3.2.0":
   version: 3.2.0
   resolution: "please-upgrade-node@npm:3.2.0"