diff --git a/.eslintignore b/.eslintignore index ef1db79c0..3458b8299 100644 --- a/.eslintignore +++ b/.eslintignore @@ -38,6 +38,10 @@ packages/app-clipper/popup/config/webpack.config.js packages/app-clipper/popup/node_modules packages/app-clipper/popup/scripts/build.js packages/app-desktop/build/ +packages/app-desktop/test-results/ +packages/app-desktop/playwright-report/ +packages/app-desktop/playwright/.cache/ +packages/app-desktop/integration-tests/test-profile/ packages/app-desktop/dist packages/app-desktop/gui/NoteEditor/NoteBody/TinyMCE/plugins/lists.js packages/app-desktop/gui/NoteEditor/NoteBody/TinyMCE/supportedLocales.js @@ -371,6 +375,13 @@ packages/app-desktop/gui/style/StyledTextInput.js packages/app-desktop/gui/utils/NoteListUtils.js packages/app-desktop/gui/utils/convertToScreenCoordinates.js packages/app-desktop/gui/utils/loadScript.js +packages/app-desktop/integration-tests/main.spec.js +packages/app-desktop/integration-tests/models/MainScreen.js +packages/app-desktop/integration-tests/models/NoteEditorScreen.js +packages/app-desktop/integration-tests/models/SettingsScreen.js +packages/app-desktop/integration-tests/util/activateMainMenuItem.js +packages/app-desktop/integration-tests/util/test.js +packages/app-desktop/playwright.config.js packages/app-desktop/plugins/GotoAnything.js packages/app-desktop/services/bridge.js packages/app-desktop/services/commands/stateToWhenClauseContext.js diff --git a/.github/workflows/github-actions-main.yml b/.github/workflows/github-actions-main.yml index 91d672734..4861b9a30 100644 --- a/.github/workflows/github-actions-main.yml +++ b/.github/workflows/github-actions-main.yml @@ -55,6 +55,9 @@ jobs: sudo apt-get install -y libsecret-1-dev sudo apt-get install -y translate-toolkit sudo apt-get install -y rsync + # Provides a virtual display on Linux. Used for Playwright integration + # testing. + sudo apt-get install -y xvfb - name: Install Docker Engine # if: runner.os == 'Linux' && startsWith(github.ref, 'refs/tags/server-v') diff --git a/.gitignore b/.gitignore index d06084d6f..7c8b15a15 100644 --- a/.gitignore +++ b/.gitignore @@ -357,6 +357,13 @@ packages/app-desktop/gui/style/StyledTextInput.js packages/app-desktop/gui/utils/NoteListUtils.js packages/app-desktop/gui/utils/convertToScreenCoordinates.js packages/app-desktop/gui/utils/loadScript.js +packages/app-desktop/integration-tests/main.spec.js +packages/app-desktop/integration-tests/models/MainScreen.js +packages/app-desktop/integration-tests/models/NoteEditorScreen.js +packages/app-desktop/integration-tests/models/SettingsScreen.js +packages/app-desktop/integration-tests/util/activateMainMenuItem.js +packages/app-desktop/integration-tests/util/test.js +packages/app-desktop/playwright.config.js packages/app-desktop/plugins/GotoAnything.js packages/app-desktop/services/bridge.js packages/app-desktop/services/commands/stateToWhenClauseContext.js diff --git a/packages/app-desktop/.gitignore b/packages/app-desktop/.gitignore index 3b065cae4..5edd796e1 100644 --- a/packages/app-desktop/.gitignore +++ b/packages/app-desktop/.gitignore @@ -14,3 +14,7 @@ style.min.css build/lib/ vendor/* !vendor/loadEmojiLib.js +test-results/ +playwright-report/ +playwright/.cache/ +integration-tests/test-profile/ diff --git a/packages/app-desktop/integration-tests/README.md b/packages/app-desktop/integration-tests/README.md new file mode 100644 index 000000000..0258c50e9 --- /dev/null +++ b/packages/app-desktop/integration-tests/README.md @@ -0,0 +1,17 @@ +# Integration tests + +The integration tests in this directory can be run with `yarn playwright test`. + +- Tests use a `test-profile` directory that should be re-created before every test. +- Only one Electron application should be instantiated per test file. +- Files in the `models/` directory follow [the page object model](https://playwright.dev/docs/pom). + +# References + +The following sources are helpful for designing and implementing Electron integration tests +with Playwright: +- [A setup guide from an organisation that uses Playwright](https://dev.to/kubeshop/testing-electron-apps-with-playwright-3f89) + and [that organisation's test suite](https://github.com/kubeshop/monokle/blob/main/tests/base.test.ts). +- [The Playwright ElectronApp docs](https://playwright.dev/docs/api/class-electronapplication) +- [Electron Playwright example repository](https://github.com/spaceagetv/electron-playwright-example) +- [Playwright best practices](https://playwright.dev/docs/best-practices) diff --git a/packages/app-desktop/integration-tests/main.spec.ts b/packages/app-desktop/integration-tests/main.spec.ts new file mode 100644 index 000000000..16b7a9b75 --- /dev/null +++ b/packages/app-desktop/integration-tests/main.spec.ts @@ -0,0 +1,79 @@ +import { test, expect } from './util/test'; +import MainScreen from './models/MainScreen'; +import activateMainMenuItem from './util/activateMainMenuItem'; +import SettingsScreen from './models/SettingsScreen'; + + +test.describe('main', () => { + test('app should launch', async ({ mainWindow }) => { + // A window should open with the correct title + expect(await mainWindow.title()).toMatch(/^Joplin/); + + const mainPage = new MainScreen(mainWindow); + await mainPage.waitFor(); + }); + + test('should be able to create and edit a new note', async ({ mainWindow }) => { + const mainScreen = new MainScreen(mainWindow); + await mainScreen.newNoteButton.click(); + + const editor = mainScreen.noteEditor; + await editor.waitFor(); + + // Wait for the title input to have the correct placeholder + await mainWindow.locator('input[placeholder^="Creating new note"]').waitFor(); + + // Fill the title + await editor.noteTitleInput.click(); + await editor.noteTitleInput.fill('Test note'); + + // Note list should contain the new note + await expect(mainScreen.noteListContainer.getByText('Test note')).toBeVisible(); + + // Focus the editor + await editor.codeMirrorEditor.click(); + + // Type some text + await mainWindow.keyboard.type('# Test note!'); + await mainWindow.keyboard.press('Enter'); + await mainWindow.keyboard.press('Enter'); + await mainWindow.keyboard.type('New note content!'); + + // Should render + const viewerFrame = editor.getNoteViewerIframe(); + await expect(viewerFrame.locator('h1')).toHaveText('Test note!'); + }); + + test('should be possible to remove sort order buttons in settings', async ({ electronApp, mainWindow }) => { + const mainScreen = new MainScreen(mainWindow); + await mainScreen.waitFor(); + + // Sort order buttons should be visible by default + await expect(mainScreen.noteListContainer.locator('[title^="Toggle sort order"]')).toBeVisible(); + + // Open settings (check both labels so that this works on MacOS) + expect( + await activateMainMenuItem(electronApp, 'Preferences...') || await activateMainMenuItem(electronApp, 'Options'), + ).toBe(true); + + // Should be on the settings screen + const settingsScreen = new SettingsScreen(mainWindow); + await settingsScreen.waitFor(); + + // Open the appearance tab + await settingsScreen.appearanceTabButton.click(); + + // Find the sort order visible checkbox + const sortOrderVisibleCheckbox = mainWindow.getByLabel(/^Show sort order/); + + await expect(sortOrderVisibleCheckbox).toBeChecked(); + await sortOrderVisibleCheckbox.click(); + await expect(sortOrderVisibleCheckbox).not.toBeChecked(); + + // Save settings & close + await settingsScreen.okayButton.click(); + await mainScreen.waitFor(); + + await expect(mainScreen.noteListContainer.locator('[title^="Toggle sort order"]')).not.toBeVisible(); + }); +}); diff --git a/packages/app-desktop/integration-tests/models/MainScreen.ts b/packages/app-desktop/integration-tests/models/MainScreen.ts new file mode 100644 index 000000000..a82eb9540 --- /dev/null +++ b/packages/app-desktop/integration-tests/models/MainScreen.ts @@ -0,0 +1,20 @@ +import { Page, Locator } from '@playwright/test'; +import NoteEditorScreen from './NoteEditorScreen'; + +export default class MainScreen { + public readonly newNoteButton: Locator; + public readonly noteListContainer: Locator; + public readonly noteEditor: NoteEditorScreen; + + public constructor(page: Page) { + this.newNoteButton = page.locator('.new-note-button'); + this.noteListContainer = page.locator('.rli-noteList'); + this.noteEditor = new NoteEditorScreen(page); + } + + public async waitFor() { + await this.newNoteButton.waitFor(); + await this.noteEditor.waitFor(); + await this.noteListContainer.waitFor(); + } +} diff --git a/packages/app-desktop/integration-tests/models/NoteEditorScreen.ts b/packages/app-desktop/integration-tests/models/NoteEditorScreen.ts new file mode 100644 index 000000000..bed36fb35 --- /dev/null +++ b/packages/app-desktop/integration-tests/models/NoteEditorScreen.ts @@ -0,0 +1,26 @@ + +import { Locator, Page } from '@playwright/test'; + +export default class NoteEditorPage { + public readonly codeMirrorEditor: Locator; + public readonly noteTitleInput: Locator; + private readonly containerLocator: Locator; + + public constructor(private readonly page: Page) { + this.containerLocator = page.locator('.rli-editor'); + this.codeMirrorEditor = this.containerLocator.locator('.codeMirrorEditor'); + this.noteTitleInput = this.containerLocator.locator('.title-input'); + } + + public getNoteViewerIframe() { + // The note viewer can change content when the note re-renders. As such, + // a new locator needs to be created after re-renders (and this can't be a + // static property). + return this.page.frame({ url: /.*note-viewer[/\\]index.html.*/ }); + } + + public async waitFor() { + await this.codeMirrorEditor.waitFor(); + await this.noteTitleInput.waitFor(); + } +} diff --git a/packages/app-desktop/integration-tests/models/SettingsScreen.ts b/packages/app-desktop/integration-tests/models/SettingsScreen.ts new file mode 100644 index 000000000..f6d671304 --- /dev/null +++ b/packages/app-desktop/integration-tests/models/SettingsScreen.ts @@ -0,0 +1,17 @@ + +import { Page, Locator } from '@playwright/test'; + +export default class SettingsScreen { + public readonly okayButton: Locator; + public readonly appearanceTabButton: Locator; + + public constructor(page: Page) { + this.okayButton = page.locator('button', { hasText: 'OK' }); + this.appearanceTabButton = page.getByText('Appearance'); + } + + public async waitFor() { + await this.okayButton.waitFor(); + await this.appearanceTabButton.waitFor(); + } +} diff --git a/packages/app-desktop/integration-tests/run-ci.sh b/packages/app-desktop/integration-tests/run-ci.sh new file mode 100755 index 000000000..071d4a3f2 --- /dev/null +++ b/packages/app-desktop/integration-tests/run-ci.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +echo "Running desktop integration tests..." + +export CI=true + +if test "$RUNNER_OS" = "Linux" ; then + # The Ubuntu Github CI doesn't have a display server. + # Start a virtual one with xvfb-run. + xvfb-run -- yarn run playwright test +else + yarn run playwright test +fi diff --git a/packages/app-desktop/integration-tests/util/activateMainMenuItem.ts b/packages/app-desktop/integration-tests/util/activateMainMenuItem.ts new file mode 100644 index 000000000..b419170b1 --- /dev/null +++ b/packages/app-desktop/integration-tests/util/activateMainMenuItem.ts @@ -0,0 +1,36 @@ + +import type { ElectronApplication } from '@playwright/test'; +import type { MenuItem } from 'electron'; + + +// Roughly based on +// https://github.com/spaceagetv/electron-playwright-helpers/blob/main/src/menu_helpers.ts + +// `menuItemPath` should be a list of menu labels (e.g. [["&JoplinMainMenu", "&File"], "Synchronise"]). +const activateMainMenuItem = (electronApp: ElectronApplication, menuItemLabel: string) => { + return electronApp.evaluate(async ({ Menu }, menuItemLabel) => { + const activateItemInSubmenu = (submenu: MenuItem[]) => { + for (const item of submenu) { + if (item.label === menuItemLabel && item.visible) { + // Found! + item.click(); + return true; + } else if (item.submenu) { + const foundItem = activateItemInSubmenu(item.submenu.items); + + if (foundItem) { + return true; + } + } + } + + // No item found + return false; + }; + + const appMenu = Menu.getApplicationMenu(); + return activateItemInSubmenu(appMenu.items); + }, menuItemLabel); +}; + +export default activateMainMenuItem; diff --git a/packages/app-desktop/integration-tests/util/test.ts b/packages/app-desktop/integration-tests/util/test.ts new file mode 100644 index 000000000..727aee539 --- /dev/null +++ b/packages/app-desktop/integration-tests/util/test.ts @@ -0,0 +1,45 @@ +import { resolve, join, dirname } from 'path'; +import { remove, mkdirp } from 'fs-extra'; +import { _electron as electron, Page, ElectronApplication, test as base } from '@playwright/test'; +import uuid from '@joplin/lib/uuid'; + + + +type JoplinFixtures = { + electronApp: ElectronApplication; + mainWindow: Page; +}; + +// A custom fixture that loads an electron app. See +// https://playwright.dev/docs/test-fixtures + +export const test = base.extend({ + // Playwright fails if we don't use the object destructuring + // pattern in the first argument. + // + // See https://github.com/microsoft/playwright/issues/8798 + // + // eslint-disable-next-line no-empty-pattern + electronApp: async ({ }, use) => { + const profilePath = resolve(join(dirname(__dirname), 'test-profile')); + const profileSubdir = join(profilePath, uuid.createNano()); + await mkdirp(profileSubdir); + + const startupArgs = ['main.js', '--env', 'dev', '--profile', profileSubdir]; + const electronApp = await electron.launch({ args: startupArgs }); + + await use(electronApp); + + await electronApp.firstWindow(); + await electronApp.close(); + await remove(profileSubdir); + }, + + mainWindow: async ({ electronApp }, use) => { + const window = await electronApp.firstWindow(); + await use(window); + }, +}); + + +export { expect } from '@playwright/test'; diff --git a/packages/app-desktop/package.json b/packages/app-desktop/package.json index ca4076691..525437a37 100644 --- a/packages/app-desktop/package.json +++ b/packages/app-desktop/package.json @@ -14,7 +14,8 @@ "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json", "start": "gulp build && electron . --env dev --log-level debug --open-dev-tools", "test": "jest", - "test-ci": "yarn test", + "test-ui": "playwright test", + "test-ci": "yarn test && sh ./integration-tests/run-ci.sh", "renameReleaseAssets": "node tools/renameReleaseAssets.js" }, "repository": { @@ -116,6 +117,7 @@ "devDependencies": { "@electron/rebuild": "3.3.0", "@joplin/tools": "~2.13", + "@playwright/test": "1.38.1", "@testing-library/react-hooks": "8.0.1", "@types/jest": "29.5.4", "@types/node": "18.17.18", diff --git a/packages/app-desktop/playwright.config.ts b/packages/app-desktop/playwright.config.ts new file mode 100644 index 000000000..87fefe4ca --- /dev/null +++ b/packages/app-desktop/playwright.config.ts @@ -0,0 +1,34 @@ +import { defineConfig } from '@playwright/test'; + +// See https://playwright.dev/docs/test-configuration. +export default defineConfig({ + testDir: './integration-tests', + + // Only match .ts files (no compiled .js files) + testMatch: '*.spec.ts', + + // Allow running tests in parallel (note: each Joplin instance + // is given its own profile directory). + fullyParallel: true, + + // Fail the build on CI if you accidentally left test.only in the source code. + forbidOnly: !!process.env.CI, + + // Retry on CI only + retries: process.env.CI ? 2 : 0, + + // Opt out of parallel tests on CI + workers: process.env.CI ? 1 : undefined, + + // Reporter to use. See https://playwright.dev/docs/test-reporters + reporter: process.env.CI ? 'line' : 'html', + + // Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. + use: { + // Base URL to use in actions like `await page.goto('/')`. + // baseURL: 'http://127.0.0.1:3000', + + // Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer + trace: 'on-first-retry', + }, +}); diff --git a/yarn.lock b/yarn.lock index 201282399..53048ca3a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4661,6 +4661,7 @@ __metadata: "@joplin/renderer": ~2.13 "@joplin/tools": ~2.13 "@joplin/utils": ~2.13 + "@playwright/test": 1.38.1 "@testing-library/react-hooks": 8.0.1 "@types/jest": 29.5.4 "@types/mustache": 4.2.2 @@ -6793,6 +6794,17 @@ __metadata: languageName: node linkType: hard +"@playwright/test@npm:1.38.1": + version: 1.38.1 + resolution: "@playwright/test@npm:1.38.1" + dependencies: + playwright: 1.38.1 + bin: + playwright: cli.js + checksum: c5ec0b23261fe1ef163b6234f69263bc10e7e5a3fb676c7773ffc70b87459a7ab225f57c03b9de649475771638a04c2e00d9b2739304a4dcf5d3edf20a7a4a82 + languageName: node + linkType: hard + "@popperjs/core@npm:^2.4.0": version: 2.11.0 resolution: "@popperjs/core@npm:2.11.0" @@ -17745,6 +17757,16 @@ __metadata: languageName: node linkType: hard +"fsevents@npm:2.3.2, fsevents@npm:^2.1.2, fsevents@npm:^2.3.2, fsevents@npm:~2.3.2": + version: 2.3.2 + resolution: "fsevents@npm:2.3.2" + dependencies: + node-gyp: latest + checksum: 97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f + conditions: os=darwin + languageName: node + linkType: hard + "fsevents@npm:^1.2.7": version: 1.2.13 resolution: "fsevents@npm:1.2.13" @@ -17756,12 +17778,11 @@ __metadata: languageName: node linkType: hard -"fsevents@npm:^2.1.2, fsevents@npm:^2.3.2, fsevents@npm:~2.3.2": +"fsevents@patch:fsevents@2.3.2#~builtin, fsevents@patch:fsevents@^2.1.2#~builtin, fsevents@patch:fsevents@^2.3.2#~builtin, fsevents@patch:fsevents@~2.3.2#~builtin": version: 2.3.2 - resolution: "fsevents@npm:2.3.2" + resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin::version=2.3.2&hash=df0bf1" dependencies: node-gyp: latest - checksum: 97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f conditions: os=darwin languageName: node linkType: hard @@ -17776,15 +17797,6 @@ __metadata: languageName: node linkType: hard -"fsevents@patch:fsevents@^2.1.2#~builtin, fsevents@patch:fsevents@^2.3.2#~builtin, fsevents@patch:fsevents@~2.3.2#~builtin": - version: 2.3.2 - resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin::version=2.3.2&hash=df0bf1" - dependencies: - node-gyp: latest - conditions: os=darwin - languageName: node - linkType: hard - "fullstore@npm:^1.0.0": version: 1.1.0 resolution: "fullstore@npm:1.1.0" @@ -28156,6 +28168,30 @@ __metadata: languageName: node linkType: hard +"playwright-core@npm:1.38.1": + version: 1.38.1 + resolution: "playwright-core@npm:1.38.1" + bin: + playwright-core: cli.js + checksum: 66e83fe040f309b13ad94ba39dea40ac207bfcbbc22de13141af88dbdedd64e1c4e3ce1d0cb070d4efd8050d7e579953ec3681dd8a0acf2c1cc738d9c50e545e + languageName: node + linkType: hard + +"playwright@npm:1.38.1": + version: 1.38.1 + resolution: "playwright@npm:1.38.1" + dependencies: + fsevents: 2.3.2 + playwright-core: 1.38.1 + dependenciesMeta: + fsevents: + optional: true + bin: + playwright: cli.js + checksum: 4e01d4ee52d9ccf75a80d8492829106802590721d56bff7c5957ff1f21eb3c328ee5bc3c1784a59c4b515df1b98d08ef92e4a35a807f454cd00dc481d30fadc2 + languageName: node + linkType: hard + "please-upgrade-node@npm:^3.2.0": version: 3.2.0 resolution: "please-upgrade-node@npm:3.2.0"