From 4fc786cf0b723158561e7883a38b65e0702c34de Mon Sep 17 00:00:00 2001 From: Henry Heino <46334387+personalizedrefrigerator@users.noreply.github.com> Date: Mon, 11 Dec 2023 05:58:45 -0800 Subject: [PATCH] Desktop: Resolves #7934: Add Simple Backup as a default plugin (#9360) --- .eslintignore | 14 +- .gitignore | 12 +- .npmpackagejsonlintignore | 4 +- .../services/plugins/defaultPluginsUtils.ts | 4 - packages/app-desktop/.gitignore | 3 + packages/app-desktop/app.ts | 7 +- packages/app-desktop/build/7zip/license.txt | 90 ++++++++ .../controls/plugins/PluginBox.tsx | 21 +- .../controls/plugins/PluginsStates.tsx | 26 ++- .../app-desktop/{gulpfile.js => gulpfile.ts} | 25 ++- .../integration-tests/main.spec.ts | 6 +- .../integration-tests/models/MainScreen.ts | 13 +- .../models/SettingsScreen.ts | 8 +- .../integration-tests/simpleBackup.spec.ts | 41 ++++ .../integration-tests/util/test.ts | 11 + packages/app-desktop/package.json | 14 +- .../services/plugins/plugin_index.js | 23 +- packages/app-desktop/tools/copy7Zip.ts | 40 ++++ packages/app-desktop/tsconfig.json | 4 + packages/default-plugins/.eslintignore | 2 + packages/default-plugins/.gitignore | 2 + packages/default-plugins/build.ts | 31 +++ .../default-plugins/buildDefaultPlugins.ts | 124 +++++++++++ packages/default-plugins/commands/buildAll.ts | 7 + .../default-plugins/commands/editPatch.ts | 31 +++ packages/default-plugins/package.json | 25 +++ .../io.github.jackgruber.backup.diff | 62 ++++++ .../default-plugins/pluginRepositories.json | 7 + packages/default-plugins/tsconfig.json | 12 + .../utils/getPathToPatchFileFor.ts | 9 + .../utils/readRepositoryJson.ts | 37 ++++ .../default-plugins/utils/waitForCliInput.ts | 23 ++ .../lib/services/plugins/PluginService.ts | 1 - .../defaultPlugins/defaultPluginsUtils.ts | 5 +- .../desktopDefaultPluginsInfo.ts | 4 - packages/tools/bundleDefaultPlugins.test.ts | 206 ------------------ packages/tools/bundleDefaultPlugins.ts | 96 -------- .../tasks/updateIgnoredTypeScriptBuild.js | 1 + packages/tools/setupNewRelease.ts | 1 + packages/utils/package.json | 1 + readme/dev/spec/default_plugins.md | 31 ++- yarn.lock | 48 ++-- 42 files changed, 754 insertions(+), 378 deletions(-) create mode 100644 packages/app-desktop/build/7zip/license.txt rename packages/app-desktop/{gulpfile.js => gulpfile.ts} (67%) create mode 100644 packages/app-desktop/integration-tests/simpleBackup.spec.ts create mode 100644 packages/app-desktop/tools/copy7Zip.ts create mode 100644 packages/default-plugins/.eslintignore create mode 100644 packages/default-plugins/.gitignore create mode 100644 packages/default-plugins/build.ts create mode 100644 packages/default-plugins/buildDefaultPlugins.ts create mode 100644 packages/default-plugins/commands/buildAll.ts create mode 100644 packages/default-plugins/commands/editPatch.ts create mode 100644 packages/default-plugins/package.json create mode 100644 packages/default-plugins/plugin-patches/io.github.jackgruber.backup.diff create mode 100644 packages/default-plugins/pluginRepositories.json create mode 100644 packages/default-plugins/tsconfig.json create mode 100644 packages/default-plugins/utils/getPathToPatchFileFor.ts create mode 100644 packages/default-plugins/utils/readRepositoryJson.ts create mode 100644 packages/default-plugins/utils/waitForCliInput.ts delete mode 100644 packages/tools/bundleDefaultPlugins.test.ts delete mode 100644 packages/tools/bundleDefaultPlugins.ts diff --git a/.eslintignore b/.eslintignore index 79f0720e42..ac0b10883c 100644 --- a/.eslintignore +++ b/.eslintignore @@ -59,6 +59,8 @@ packages/app-mobile/locales packages/app-mobile/node_modules packages/app-mobile/pluginAssets/ packages/fork-* +packages/default-plugins/plugin-base-repo/ +packages/default-plugins/plugin-sources/ packages/htmlpack/dist/ packages/lib/assets/ packages/lib/lib/lib.js @@ -380,10 +382,12 @@ packages/app-desktop/gui/style/StyledTextInput.js packages/app-desktop/gui/utils/NoteListUtils.js packages/app-desktop/gui/utils/convertToScreenCoordinates.js packages/app-desktop/gui/utils/loadScript.js +packages/app-desktop/gulpfile.js packages/app-desktop/integration-tests/main.spec.js packages/app-desktop/integration-tests/models/MainScreen.js packages/app-desktop/integration-tests/models/NoteEditorScreen.js packages/app-desktop/integration-tests/models/SettingsScreen.js +packages/app-desktop/integration-tests/simpleBackup.spec.js packages/app-desktop/integration-tests/util/activateMainMenuItem.js packages/app-desktop/integration-tests/util/createStartupArgs.js packages/app-desktop/integration-tests/util/firstNonDevToolsWindow.js @@ -415,6 +419,7 @@ packages/app-desktop/services/sortOrder/PerFolderSortOrderService.js packages/app-desktop/services/sortOrder/notesSortOrderUtils.test.js packages/app-desktop/services/sortOrder/notesSortOrderUtils.js packages/app-desktop/services/spellChecker/SpellCheckerServiceDriverNative.js +packages/app-desktop/tools/copy7Zip.js packages/app-desktop/tools/notarizeMacApp.js packages/app-desktop/tools/renameReleaseAssets.js packages/app-desktop/utils/checkForUpdatesUtils.test.js @@ -530,6 +535,13 @@ packages/app-mobile/utils/fs-driver/runOnDeviceTests.js packages/app-mobile/utils/setupNotifications.js packages/app-mobile/utils/shareHandler.js packages/app-mobile/utils/types.js +packages/default-plugins/build.js +packages/default-plugins/buildDefaultPlugins.js +packages/default-plugins/commands/buildAll.js +packages/default-plugins/commands/editPatch.js +packages/default-plugins/utils/getPathToPatchFileFor.js +packages/default-plugins/utils/readRepositoryJson.js +packages/default-plugins/utils/waitForCliInput.js packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.test.js packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.js packages/editor/CodeMirror/CodeMirror5Emulation/Decorator.js @@ -1013,8 +1025,6 @@ packages/tools/build-translation.js packages/tools/build-welcome.js packages/tools/buildServerDocker.test.js packages/tools/buildServerDocker.js -packages/tools/bundleDefaultPlugins.test.js -packages/tools/bundleDefaultPlugins.js packages/tools/checkIgnoredFiles.js packages/tools/checkLibPaths.test.js packages/tools/checkLibPaths.js diff --git a/.gitignore b/.gitignore index 664cfe5309..e5c289eec8 100644 --- a/.gitignore +++ b/.gitignore @@ -362,10 +362,12 @@ packages/app-desktop/gui/style/StyledTextInput.js packages/app-desktop/gui/utils/NoteListUtils.js packages/app-desktop/gui/utils/convertToScreenCoordinates.js packages/app-desktop/gui/utils/loadScript.js +packages/app-desktop/gulpfile.js packages/app-desktop/integration-tests/main.spec.js packages/app-desktop/integration-tests/models/MainScreen.js packages/app-desktop/integration-tests/models/NoteEditorScreen.js packages/app-desktop/integration-tests/models/SettingsScreen.js +packages/app-desktop/integration-tests/simpleBackup.spec.js packages/app-desktop/integration-tests/util/activateMainMenuItem.js packages/app-desktop/integration-tests/util/createStartupArgs.js packages/app-desktop/integration-tests/util/firstNonDevToolsWindow.js @@ -397,6 +399,7 @@ packages/app-desktop/services/sortOrder/PerFolderSortOrderService.js packages/app-desktop/services/sortOrder/notesSortOrderUtils.test.js packages/app-desktop/services/sortOrder/notesSortOrderUtils.js packages/app-desktop/services/spellChecker/SpellCheckerServiceDriverNative.js +packages/app-desktop/tools/copy7Zip.js packages/app-desktop/tools/notarizeMacApp.js packages/app-desktop/tools/renameReleaseAssets.js packages/app-desktop/utils/checkForUpdatesUtils.test.js @@ -512,6 +515,13 @@ packages/app-mobile/utils/fs-driver/runOnDeviceTests.js packages/app-mobile/utils/setupNotifications.js packages/app-mobile/utils/shareHandler.js packages/app-mobile/utils/types.js +packages/default-plugins/build.js +packages/default-plugins/buildDefaultPlugins.js +packages/default-plugins/commands/buildAll.js +packages/default-plugins/commands/editPatch.js +packages/default-plugins/utils/getPathToPatchFileFor.js +packages/default-plugins/utils/readRepositoryJson.js +packages/default-plugins/utils/waitForCliInput.js packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.test.js packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.js packages/editor/CodeMirror/CodeMirror5Emulation/Decorator.js @@ -995,8 +1005,6 @@ packages/tools/build-translation.js packages/tools/build-welcome.js packages/tools/buildServerDocker.test.js packages/tools/buildServerDocker.js -packages/tools/bundleDefaultPlugins.test.js -packages/tools/bundleDefaultPlugins.js packages/tools/checkIgnoredFiles.js packages/tools/checkLibPaths.test.js packages/tools/checkLibPaths.js diff --git a/.npmpackagejsonlintignore b/.npmpackagejsonlintignore index cfed56947f..a22d546d1b 100644 --- a/.npmpackagejsonlintignore +++ b/.npmpackagejsonlintignore @@ -1,3 +1,5 @@ packages/app-clipper/popup/ packages/app-cli/tests/support/plugins/ -packages/doc-builder/ \ No newline at end of file +packages/doc-builder/ +packages/default-plugins/plugin-base-repo/ +packages/default-plugins/plugin-sources/ \ No newline at end of file diff --git a/packages/app-cli/tests/services/plugins/defaultPluginsUtils.ts b/packages/app-cli/tests/services/plugins/defaultPluginsUtils.ts index 58b51c1c78..8508befc94 100644 --- a/packages/app-cli/tests/services/plugins/defaultPluginsUtils.ts +++ b/packages/app-cli/tests/services/plugins/defaultPluginsUtils.ts @@ -189,13 +189,11 @@ describe('defaultPluginsUtils', () => { const defaultPluginsInfo: DefaultPluginsInfo = { 'io.github.jackgruber.backup': { - version: '1.0.2', settings: { 'path': `${Setting.value('profileDir')}`, }, }, 'plugin.calebjohn.rich-markdown': { - version: '0.8.3', }, }; @@ -245,14 +243,12 @@ describe('defaultPluginsUtils', () => { const defaultPluginsInfo: DefaultPluginsInfo = { 'io.github.jackgruber.backup': { - version: '1.0.2', settings: { 'path': `${Setting.value('profileDir')}`, 'missing-key1': 'someValue', }, }, 'plugin.calebjohn.rich-markdown': { - version: '0.8.3', settings: { 'missing-key2': 'someValue', }, diff --git a/packages/app-desktop/.gitignore b/packages/app-desktop/.gitignore index 5edd796e18..5ab3baf156 100644 --- a/packages/app-desktop/.gitignore +++ b/packages/app-desktop/.gitignore @@ -18,3 +18,6 @@ test-results/ playwright-report/ playwright/.cache/ integration-tests/test-profile/ +build/defaultPlugins/ +build/7zip/7za +build/7zip/7za.exe diff --git a/packages/app-desktop/app.ts b/packages/app-desktop/app.ts index b5f3a527b2..87a885e3cb 100644 --- a/packages/app-desktop/app.ts +++ b/packages/app-desktop/app.ts @@ -29,7 +29,7 @@ import { reg } from '@joplin/lib/registry'; const packageInfo: PackageInfo = require('./packageInfo.js'); import DecryptionWorker from '@joplin/lib/services/DecryptionWorker'; import ClipperServer from '@joplin/lib/ClipperServer'; -const { webFrame } = require('electron'); +import { ipcRenderer, webFrame } from 'electron'; const Menu = bridge().Menu; const PluginManager = require('@joplin/lib/services/PluginManager'); import RevisionService from '@joplin/lib/services/RevisionService'; @@ -333,6 +333,11 @@ class Application extends BaseApplication { type: 'STARTUP_PLUGINS_LOADED', value: true, }); + + // Sends an event to the main process -- this is used by the Playwright + // tests to wait for plugins to load. + ipcRenderer.send('startup-plugins-loaded'); + setSettingsForDefaultPlugins(getDefaultPluginsInfo()); } }, 500); diff --git a/packages/app-desktop/build/7zip/license.txt b/packages/app-desktop/build/7zip/license.txt new file mode 100644 index 0000000000..0cd30744ef --- /dev/null +++ b/packages/app-desktop/build/7zip/license.txt @@ -0,0 +1,90 @@ +7-Zip +~~~~~ +License for use and distribution +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +7-Zip Copyright (C) 1999-2023 Igor Pavlov. + +The licenses for files are: + +1) 7z.dll: + - The "GNU LGPL" as main license for most of the code + - The "GNU LGPL" with "unRAR license restriction" for some code + - The "BSD 3-clause License" for some code +2) All other files: the "GNU LGPL". + +Redistributions in binary form must reproduce related license information from this file. + +Note: +You can use 7-Zip on any computer, including a computer in a commercial +organization. You don't need to register or pay for 7-Zip. + + +GNU LGPL information +-------------------- + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You can receive a copy of the GNU Lesser General Public License from +http://www.gnu.org/ + + + + +BSD 3-clause License +-------------------- + +The "BSD 3-clause License" is used for the code in 7z.dll that implements LZFSE data decompression. +That code was derived from the code in the "LZFSE compression library" developed by Apple Inc, +that also uses the "BSD 3-clause License": + +---- +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +---- + + + + +unRAR license restriction +------------------------- + +The decompression engine for RAR archives was developed using source +code of unRAR program. +All copyrights to original unRAR code are owned by Alexander Roshal. + +The license for original unRAR code has the following restriction: + + The unRAR sources cannot be used to re-create the RAR compression algorithm, + which is proprietary. Distribution of modified unRAR sources in separate form + or as a part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + + +-- +Igor Pavlov diff --git a/packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginBox.tsx b/packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginBox.tsx index 34c63dbcf4..bb257d6423 100644 --- a/packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginBox.tsx +++ b/packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginBox.tsx @@ -60,7 +60,7 @@ const CellRoot = styled.div<{ isCompatible: boolean }>` box-sizing: border-box; background-color: ${props => props.theme.backgroundColor}; flex-direction: column; - align-items: flex-start; + align-items: stretch; padding: 15px; border: 1px solid ${props => props.theme.dividerColor}; border-radius: 6px; @@ -96,12 +96,15 @@ const NeedUpgradeMessage = styled.span` font-size: ${props => props.theme.fontSize}px; `; -const DevModeLabel = styled.div` +const BoxedLabel = styled.div` border: 1px solid ${props => props.theme.color}; border-radius: 4px; padding: 4px 6px; font-size: ${props => props.theme.fontSize * 0.75}px; color: ${props => props.theme.color}; + flex-grow: 0; + height: min-content; + margin-top: auto; `; const StyledNameAndVersion = styled.div<{ mb: any }>` @@ -170,7 +173,7 @@ export default function(props: Props) { if (!props.onToggle) return null; if (item.devMode) { - return DEV; + return DEV; } return ; } + const renderDefaultPluginLabel = () => { + // Built-in plugins can only be disabled + if (item.manifest._built_in) { + return ( + {_('Built in')} + ); + } + + return null; + }; + function renderFooter() { if (item.devMode) return null; @@ -236,6 +250,7 @@ export default function(props: Props) { {renderInstallButton()} {renderUpdateButton()}
+ {renderDefaultPluginLabel()} ); } diff --git a/packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginsStates.tsx b/packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginsStates.tsx index ec44507a3e..edf907cffb 100644 --- a/packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginsStates.tsx +++ b/packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginsStates.tsx @@ -147,11 +147,18 @@ export default function(props: Props) { let cancelled = false; async function fetchPluginIds() { - const pluginIds = await repoApi().canBeUpdatedPlugins(pluginItems.map(p => p.manifest), pluginService.appVersion); + // Built-in plugins can't be updated from the main repoApi + const nonDefaultPlugins = pluginItems + .map(p => p.manifest) + .filter(manifest => !manifest._built_in); + + const pluginIds = await repoApi().canBeUpdatedPlugins(nonDefaultPlugins, pluginService.appVersion); if (cancelled) return; + const conv: Record = {}; - // eslint-disable-next-line github/array-foreach -- Old code before rule was applied - pluginIds.forEach(id => conv[id] = true); + for (const id of pluginIds) { + conv[id] = true; + } setCanBeUpdatedPluginIds(conv); } @@ -281,10 +288,17 @@ export default function(props: Props) { ); } else { + const nonDefaultPlugins = pluginItems.filter(item => !item.manifest._built_in); + const defaultPlugins = pluginItems.filter(item => item.manifest._built_in); return ( - - {renderCells(pluginItems)} - + <> + + {renderCells(nonDefaultPlugins)} + + + {renderCells(defaultPlugins)} + + ); } } diff --git a/packages/app-desktop/gulpfile.js b/packages/app-desktop/gulpfile.ts similarity index 67% rename from packages/app-desktop/gulpfile.js rename to packages/app-desktop/gulpfile.ts index 1fec0b95fc..871cec44d3 100644 --- a/packages/app-desktop/gulpfile.js +++ b/packages/app-desktop/gulpfile.ts @@ -2,6 +2,8 @@ const gulp = require('gulp'); const utils = require('@joplin/tools/gulp/utils'); const compileSass = require('@joplin/tools/compileSass'); const compilePackageInfo = require('@joplin/tools/compilePackageInfo'); +import buildDefaultPlugins from '@joplin/default-plugins/commands/buildAll'; +import copy7Zip from './tools/copy7Zip'; const tasks = { compileScripts: { @@ -24,6 +26,17 @@ const tasks = { electronBuilder: { fn: require('./tools/electronBuilder.js'), }, + copyDefaultPluginsAssets: { + fn: async () => { + await copy7Zip(); + }, + }, + buildDefaultPlugins: { + fn: async () => { + const outputDir = `${__dirname}/build/defaultPlugins/`; + await buildDefaultPlugins(outputDir); + }, + }, tsc: require('@joplin/tools/gulp/tasks/tsc'), updateIgnoredTypeScriptBuild: require('@joplin/tools/gulp/tasks/updateIgnoredTypeScriptBuild'), buildCommandIndex: require('@joplin/tools/gulp/tasks/buildCommandIndex'), @@ -39,7 +52,7 @@ const tasks = { utils.registerGulpTasks(gulp, tasks); -const buildParallel = [ +const buildBeforeStartParallel = [ 'compileScripts', 'compilePackageInfo', 'copyPluginAssets', @@ -49,4 +62,12 @@ const buildParallel = [ 'compileSass', ]; -gulp.task('build', gulp.parallel(...buildParallel)); +gulp.task('before-start', gulp.parallel(...buildBeforeStartParallel)); + +const buildAllSequential = [ + 'before-start', + 'copyDefaultPluginsAssets', + 'buildDefaultPlugins', +]; + +gulp.task('build', gulp.series(buildAllSequential)); diff --git a/packages/app-desktop/integration-tests/main.spec.ts b/packages/app-desktop/integration-tests/main.spec.ts index b7238b28fa..f6a2a9eb74 100644 --- a/packages/app-desktop/integration-tests/main.spec.ts +++ b/packages/app-desktop/integration-tests/main.spec.ts @@ -1,6 +1,5 @@ import { test, expect } from './util/test'; import MainScreen from './models/MainScreen'; -import activateMainMenuItem from './util/activateMainMenuItem'; import SettingsScreen from './models/SettingsScreen'; import { _electron as electron } from '@playwright/test'; import { writeFile } from 'fs-extra'; @@ -91,10 +90,7 @@ test.describe('main', () => { // Sort order buttons should be visible by default await expect(mainScreen.noteListContainer.locator('[title^="Toggle sort order"]')).toBeVisible(); - // Open settings (check both labels so that this works on MacOS) - expect( - await activateMainMenuItem(electronApp, 'Preferences...') || await activateMainMenuItem(electronApp, 'Options'), - ).toBe(true); + await mainScreen.openSettings(electronApp); // Should be on the settings screen const settingsScreen = new SettingsScreen(mainWindow); diff --git a/packages/app-desktop/integration-tests/models/MainScreen.ts b/packages/app-desktop/integration-tests/models/MainScreen.ts index c1015e9cf3..3ba1fb09f8 100644 --- a/packages/app-desktop/integration-tests/models/MainScreen.ts +++ b/packages/app-desktop/integration-tests/models/MainScreen.ts @@ -1,5 +1,6 @@ -import { Page, Locator } from '@playwright/test'; +import { Page, Locator, ElectronApplication } from '@playwright/test'; import NoteEditorScreen from './NoteEditorScreen'; +import activateMainMenuItem from '../util/activateMainMenuItem'; export default class MainScreen { public readonly newNoteButton: Locator; @@ -33,4 +34,14 @@ export default class MainScreen { return this.noteEditor; } + + public async openSettings(electronApp: ElectronApplication) { + // Check both labels so this works on MacOS + const openedWithPreferences = await activateMainMenuItem(electronApp, 'Preferences...'); + const openedWithOptions = await activateMainMenuItem(electronApp, 'Options'); + + if (!openedWithOptions && !openedWithPreferences) { + throw new Error('Unable to find settings menu item in application menus.'); + } + } } diff --git a/packages/app-desktop/integration-tests/models/SettingsScreen.ts b/packages/app-desktop/integration-tests/models/SettingsScreen.ts index f6d6713042..dcc0e97b1f 100644 --- a/packages/app-desktop/integration-tests/models/SettingsScreen.ts +++ b/packages/app-desktop/integration-tests/models/SettingsScreen.ts @@ -5,9 +5,13 @@ export default class SettingsScreen { public readonly okayButton: Locator; public readonly appearanceTabButton: Locator; - public constructor(page: Page) { + public constructor(private page: Page) { this.okayButton = page.locator('button', { hasText: 'OK' }); - this.appearanceTabButton = page.getByText('Appearance'); + this.appearanceTabButton = this.getTabLocator('Appearance'); + } + + public getTabLocator(tabName: string) { + return this.page.locator('a[role="tab"] > span', { hasText: tabName }); } public async waitFor() { diff --git a/packages/app-desktop/integration-tests/simpleBackup.spec.ts b/packages/app-desktop/integration-tests/simpleBackup.spec.ts new file mode 100644 index 0000000000..1bf70e7893 --- /dev/null +++ b/packages/app-desktop/integration-tests/simpleBackup.spec.ts @@ -0,0 +1,41 @@ +import { test, expect } from './util/test'; +import MainScreen from './models/MainScreen'; +import SettingsScreen from './models/SettingsScreen'; +import activateMainMenuItem from './util/activateMainMenuItem'; + +test.describe('simpleBackup', () => { + test('should have a section in settings', async ({ electronApp, startupPluginsLoaded, mainWindow }) => { + await startupPluginsLoaded; + + const mainScreen = new MainScreen(mainWindow); + await mainScreen.waitFor(); + + // Open settings (check both labels so that this works on MacOS) + await mainScreen.openSettings(electronApp); + + // Should be on the settings screen + const settingsScreen = new SettingsScreen(mainWindow); + await settingsScreen.waitFor(); + + const backupTab = settingsScreen.getTabLocator('Backup'); + await backupTab.waitFor(); + }); + + test('should be possible to create a backup', async ({ electronApp, startupPluginsLoaded, mainWindow }) => { + await startupPluginsLoaded; + + const mainScreen = new MainScreen(mainWindow); + await mainScreen.waitFor(); + + // Backups should work + expect(await activateMainMenuItem(electronApp, 'Create backup')).toBe(true); + + const successDialog = mainWindow.locator('iframe[id$=backup-backupDialog]'); + await successDialog.waitFor(); + + // Should report success + const dialogContentLocator = successDialog.frameLocator(':scope'); + await dialogContentLocator.getByText('Backup completed').waitFor(); + }); +}); + diff --git a/packages/app-desktop/integration-tests/util/test.ts b/packages/app-desktop/integration-tests/util/test.ts index f3c3a2e0bd..faee4bebda 100644 --- a/packages/app-desktop/integration-tests/util/test.ts +++ b/packages/app-desktop/integration-tests/util/test.ts @@ -10,6 +10,7 @@ import firstNonDevToolsWindow from './firstNonDevToolsWindow'; type JoplinFixtures = { profileDirectory: string; electronApp: ElectronApplication; + startupPluginsLoaded: Promise; mainWindow: Page; }; @@ -43,6 +44,16 @@ export const test = base.extend({ await electronApp.close(); }, + startupPluginsLoaded: async ({ electronApp }, use) => { + const startupPluginsLoadedPromise = electronApp.evaluate(({ ipcMain }) => { + return new Promise(resolve => { + ipcMain.once('startup-plugins-loaded', () => resolve()); + }); + }); + + await use(startupPluginsLoadedPromise); + }, + mainWindow: async ({ electronApp }, use) => { const mainWindow = await firstNonDevToolsWindow(electronApp); await use(mainWindow); diff --git a/packages/app-desktop/package.json b/packages/app-desktop/package.json index ac88a93fec..438672536b 100644 --- a/packages/app-desktop/package.json +++ b/packages/app-desktop/package.json @@ -7,12 +7,11 @@ "scripts": { "dist": "yarn run electronRebuild && npx electron-builder", "build": "gulp build", - "postinstall": "yarn run build", "electronBuilder": "gulp electronBuilder", "electronRebuild": "gulp electronRebuild", "tsc": "tsc --project tsconfig.json", "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json", - "start": "gulp build && electron . --env dev --log-level debug --open-dev-tools", + "start": "gulp before-start && electron . --env dev --log-level debug --open-dev-tools", "test": "jest", "test-ui": "playwright test", "test-ci": "yarn test && sh ./integration-tests/run-ci.sh", @@ -36,7 +35,8 @@ "extraResources": [ "build/icons/**", "build/images/**", - "build/defaultPlugins/**" + "build/defaultPlugins/**", + "build/7zip/**" ], "afterAllArtifactBuild": "./generateSha512.js", "asar": true, @@ -115,7 +115,9 @@ }, "homepage": "https://github.com/laurent22/joplin#readme", "devDependencies": { + "7zip-bin": "5.2.0", "@electron/rebuild": "3.3.0", + "@joplin/default-plugins": "~2.13", "@joplin/tools": "~2.13", "@playwright/test": "1.39.0", "@testing-library/react-hooks": "8.0.1", @@ -133,13 +135,9 @@ "js-sha512": "0.8.0", "nan": "2.18.0", "react-test-renderer": "18.2.0", + "ts-node": "10.9.1", "typescript": "5.2.2" }, - "optionalDependencies": { - "7zip-bin-linux": "^1.0.1", - "7zip-bin-mac": "^1.0.1", - "7zip-bin-win": "^2.1.1" - }, "dependencies": { "@electron/notarize": "2.1.0", "@electron/remote": "2.0.12", diff --git a/packages/app-desktop/services/plugins/plugin_index.js b/packages/app-desktop/services/plugins/plugin_index.js index 2e3a761923..f42f1b99ca 100644 --- a/packages/app-desktop/services/plugins/plugin_index.js +++ b/packages/app-desktop/services/plugins/plugin_index.js @@ -2,6 +2,7 @@ // TODO: Not sure if that will work once packaged in Electron const sandboxProxy = require('../../vendor/lib/@joplin/lib/services/plugins/sandboxProxy.js'); const ipcRenderer = require('electron').ipcRenderer; + const nodePath = require('path'); const ipcRendererSend = (message, args) => { try { @@ -56,7 +57,27 @@ return require('../../node_modules/@joplin/lib/node_modules/sqlite3/lib/sqlite3.js'); } - if (['fs-extra'].includes(modulePath)) return require(modulePath); + if (modulePath === 'fs-extra') { + return require('fs-extra'); + } + + // 7zip-bin is required by one of the default plugins (simple-backup) + if (modulePath === '7zip-bin') { + // 7zip-bin is very large -- return the path to a version of 7zip + // copied from 7zip-bin. + const executableName = process.platform === 'win32' ? '7za.exe' : '7za'; + + let rootDir = nodePath.dirname(nodePath.dirname(__dirname)); + + // When bundled, __dirname points to a file within app.asar. The build/ directory + // is outside of app.asar, and thus, we need an extra dirname(...). + if (nodePath.basename(rootDir).startsWith('app.asar')) { + rootDir = nodePath.dirname(rootDir); + } + + const pathTo7za = nodePath.join(rootDir, 'build', '7zip', executableName); + return { path7za: nodePath.resolve(pathTo7za) }; + } throw new Error(`Module not found: ${modulePath}`); } diff --git a/packages/app-desktop/tools/copy7Zip.ts b/packages/app-desktop/tools/copy7Zip.ts new file mode 100644 index 0000000000..531dc01d73 --- /dev/null +++ b/packages/app-desktop/tools/copy7Zip.ts @@ -0,0 +1,40 @@ + +import { copy } from 'fs-extra'; +import { dirname, join } from 'path'; + +const copy7Zip = async () => { + // We allow buildin for a different architecture/platform with + // the npm_config_target_arch and npm_config_target_platform environment variables. + // + // These are the same environment variables used by yarn when downloading dependencies. + // + const targetArch = process.env['npm_config_target_arch'] || process.arch; + const targetPlatform = process.env['npm_config_target_platform'] || process.platform; + + console.info('Copying 7zip for platform', targetPlatform, 'and architecture', targetArch); + + // To use the custom architecture/platform, we copy the relevant files from 7zip-bin + // directly: + + const sevenZipBinDirectory = dirname(require.resolve('7zip-bin')); + const platformToSubdirectory: Record = { + 'win32': 'win', + 'darwin': 'mac', + 'linux': 'linux', + }; + + if (!(targetPlatform in platformToSubdirectory)) { + throw new Error(`Invalid target platform ${targetPlatform}. Must be in ${Object.keys(platformToSubdirectory)}`); + } + + const fileName = targetPlatform === 'win32' ? '7za.exe' : '7za'; + const pathTo7za = join( + sevenZipBinDirectory, platformToSubdirectory[targetPlatform], targetArch, fileName, + ); + + const rootDir = dirname(__dirname); + const outputPath = join(rootDir, 'build', '7zip', fileName); + await copy(pathTo7za, outputPath); +}; + +export default copy7Zip; diff --git a/packages/app-desktop/tsconfig.json b/packages/app-desktop/tsconfig.json index e1dd295bc7..6a82e03167 100644 --- a/packages/app-desktop/tsconfig.json +++ b/packages/app-desktop/tsconfig.json @@ -7,5 +7,9 @@ "exclude": [ "**/node_modules", "**/dist", + + // Exclude gulpfile.ts to prevent Gulp from trying to build from + // gulpfile.js. + "gulpfile.ts" ], } \ No newline at end of file diff --git a/packages/default-plugins/.eslintignore b/packages/default-plugins/.eslintignore new file mode 100644 index 0000000000..301fa9092c --- /dev/null +++ b/packages/default-plugins/.eslintignore @@ -0,0 +1,2 @@ +plugin-base-repo/ +plugin-sources/* \ No newline at end of file diff --git a/packages/default-plugins/.gitignore b/packages/default-plugins/.gitignore new file mode 100644 index 0000000000..22ceeb450b --- /dev/null +++ b/packages/default-plugins/.gitignore @@ -0,0 +1,2 @@ +built-plugins/ +plugin-sources/* \ No newline at end of file diff --git a/packages/default-plugins/build.ts b/packages/default-plugins/build.ts new file mode 100644 index 0000000000..b4eec9c294 --- /dev/null +++ b/packages/default-plugins/build.ts @@ -0,0 +1,31 @@ +import buildAll from './commands/buildAll'; +import editPatch from './commands/editPatch'; +const yargs = require('yargs'); + + +const build = () => { + yargs + .usage('$0 [args]') + .command('build ', 'build all', (yargs: any) => { + yargs.positional('outputDir', { + type: 'string', + describe: 'Path to the parent directory for built output', + }); + }, async (args: any) => { + await buildAll(args.outputDir); + process.exit(0); + }) + .command('patch ', 'Edit the patch file for the given plugin ID', (yargs: any) => { + yargs.positional('plugin', { + type: 'string', + describe: 'ID of the plugin to patch', + }); + }, async (args: any) => { + await editPatch(args.plugin, null); + process.exit(0); + }) + .help() + .argv; +}; + +build(); diff --git a/packages/default-plugins/buildDefaultPlugins.ts b/packages/default-plugins/buildDefaultPlugins.ts new file mode 100644 index 0000000000..0ccccca638 --- /dev/null +++ b/packages/default-plugins/buildDefaultPlugins.ts @@ -0,0 +1,124 @@ + +/* eslint-disable no-console */ + +import { copy, exists, remove, mkdirp, readdir, mkdtemp, readFile, writeFile } from 'fs-extra'; +import { join, resolve, basename } from 'path'; +import { tmpdir } from 'os'; +import { chdir, cwd } from 'process'; +import { execCommand } from '@joplin/utils'; +import { glob } from 'glob'; +import readRepositoryJson from './utils/readRepositoryJson'; +import waitForCliInput from './utils/waitForCliInput'; +import getPathToPatchFileFor from './utils/getPathToPatchFileFor'; + +type BeforeEachInstallCallback = (buildDir: string, pluginName: string)=> Promise; + +const buildDefaultPlugins = async (outputParentDir: string|null, beforeInstall: BeforeEachInstallCallback) => { + const pluginSourcesDir = resolve(join(__dirname, 'plugin-sources')); + const pluginRepositoryData = await readRepositoryJson(join(__dirname, 'pluginRepositories.json')); + + const originalDirectory = cwd(); + + const logStatus = (...message: string[]) => { + const blue = '\x1b[96m'; + const reset = '\x1b[0m'; + console.log(blue, ...message, reset); + }; + + for (const pluginId in pluginRepositoryData) { + const repositoryData = pluginRepositoryData[pluginId]; + + const buildDir = await mkdtemp(join(tmpdir(), 'default-plugin-build')); + try { + logStatus('Building plugin', pluginId, 'at', buildDir); + const pluginDir = resolve(join(pluginSourcesDir, pluginId)); + + // Clone the repository if not done yet + if (!(await exists(pluginDir)) || (await readdir(pluginDir)).length === 0) { + logStatus(`Cloning from repository ${repositoryData.cloneUrl}`); + await execCommand(['git', 'clone', '--', repositoryData.cloneUrl, pluginDir]); + chdir(pluginDir); + } + + chdir(pluginDir); + const currentCommitHash = (await execCommand(['git', 'rev-parse', 'HEAD~'])).trim(); + const expectedCommitHash = repositoryData.commit; + + if (currentCommitHash !== expectedCommitHash) { + logStatus(`Switching to commit ${expectedCommitHash}`); + await execCommand(['git', 'switch', repositoryData.branch]); + await execCommand(['git', 'checkout', expectedCommitHash]); + } + + logStatus('Copying repository files...'); + await copy(pluginDir, buildDir, { + filter: fileName => { + return basename(fileName) !== '.git'; + }, + }); + + chdir(buildDir); + + logStatus('Initializing repository.'); + await execCommand('git init . -b main'); + + logStatus('Marking manifest as built-in'); + const manifestFile = './src/manifest.json'; + const manifest = JSON.parse(await readFile(manifestFile, 'utf8')); + manifest._built_in = true; + await writeFile(manifestFile, JSON.stringify(manifest, undefined, '\t')); + + logStatus('Creating initial commit.'); + await execCommand('git add .'); + await execCommand(['git', 'config', 'user.name', 'Build script']); + await execCommand(['git', 'config', 'user.email', '']); + await execCommand(['git', 'commit', '-m', 'Initial commit']); + + const patchFile = getPathToPatchFileFor(pluginId); + if (await exists(patchFile)) { + logStatus('Applying patch.'); + await execCommand(['git', 'apply', patchFile]); + } + + await beforeInstall(buildDir, pluginId); + + logStatus('Installing dependencies.'); + await execCommand('npm install'); + + const jplFiles = await glob('publish/*.jpl'); + logStatus(`Found built .jpl files: ${JSON.stringify(jplFiles)}`); + + if (jplFiles.length === 0) { + throw new Error(`No published files found in ${buildDir}/publish`); + } + + if (outputParentDir !== null) { + logStatus(`Checking output directory in ${outputParentDir}`); + const outputDirectory = join(outputParentDir, pluginId); + if (await exists(outputDirectory)) { + await remove(outputDirectory); + } + await mkdirp(outputDirectory); + + const sourceFile = jplFiles[0]; + const destFile = join(outputDirectory, 'plugin.jpl'); + + logStatus(`Copying built file from ${sourceFile} to ${destFile}`); + await copy(sourceFile, destFile); + } else { + console.warn('No output directory specified. Not copying built .jpl files.'); + } + } catch (error) { + console.error(error); + console.log('Build directory', buildDir); + await waitForCliInput(); + throw error; + } finally { + chdir(originalDirectory); + await remove(buildDir); + logStatus('Removed build directory'); + } + } +}; + +export default buildDefaultPlugins; diff --git a/packages/default-plugins/commands/buildAll.ts b/packages/default-plugins/commands/buildAll.ts new file mode 100644 index 0000000000..8d02b26a97 --- /dev/null +++ b/packages/default-plugins/commands/buildAll.ts @@ -0,0 +1,7 @@ +import buildDefaultPlugins from '../buildDefaultPlugins'; + +const buildAll = (outputDirectory: string) => { + return buildDefaultPlugins(outputDirectory, async () => { }); +}; + +export default buildAll; diff --git a/packages/default-plugins/commands/editPatch.ts b/packages/default-plugins/commands/editPatch.ts new file mode 100644 index 0000000000..3294d744e6 --- /dev/null +++ b/packages/default-plugins/commands/editPatch.ts @@ -0,0 +1,31 @@ +import { execCommand } from '@joplin/utils'; +import waitForCliInput from '../utils/waitForCliInput'; +import { copy } from 'fs-extra'; +import { join } from 'path'; +import buildDefaultPlugins from '../buildDefaultPlugins'; +import getPathToPatchFileFor from '../utils/getPathToPatchFileFor'; + +const editPatch = async (targetPluginId: string, outputParentDir: string|null) => { + let patchedPlugin = false; + + await buildDefaultPlugins(outputParentDir, async (buildDir, pluginId) => { + if (pluginId !== targetPluginId) { + return; + } + + // eslint-disable-next-line no-console + console.log('Make changes to', buildDir, 'to create a patch.'); + await waitForCliInput(); + await execCommand(['sh', '-c', 'git diff -p > diff.diff']); + + await copy(join(buildDir, './diff.diff'), getPathToPatchFileFor(pluginId)); + + patchedPlugin = true; + }); + + if (!patchedPlugin) { + throw new Error(`No default plugin with ID ${targetPluginId} found!`); + } +}; + +export default editPatch; diff --git a/packages/default-plugins/package.json b/packages/default-plugins/package.json new file mode 100644 index 0000000000..cf1fea1125 --- /dev/null +++ b/packages/default-plugins/package.json @@ -0,0 +1,25 @@ +{ + "name": "@joplin/default-plugins", + "version": "2.13.0", + "description": "Default plugins bundler", + "private": true, + "scripts": { + "tsc": "tsc --project tsconfig.json", + "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json", + "patch": "ts-node build.ts patch" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/laurent22/joplin.git" + }, + "devDependencies": { + "@types/yargs": "17.0.31", + "ts-node": "10.9.1", + "typescript": "5.2.2" + }, + "dependencies": { + "@joplin/utils": "~2.13", + "fs-extra": "11.1.1", + "yargs": "17.7.2" + } +} diff --git a/packages/default-plugins/plugin-patches/io.github.jackgruber.backup.diff b/packages/default-plugins/plugin-patches/io.github.jackgruber.backup.diff new file mode 100644 index 0000000000..79d9766247 --- /dev/null +++ b/packages/default-plugins/plugin-patches/io.github.jackgruber.backup.diff @@ -0,0 +1,62 @@ +diff --git a/src/sevenZip.ts b/src/sevenZip.ts +index ef2a527..d98c777 100644 +--- a/src/sevenZip.ts ++++ b/src/sevenZip.ts +@@ -1,21 +1,21 @@ + // https://sevenzip.osdn.jp/chm/cmdline/exit_codes.htm + // https://sevenzip.osdn.jp/chm/cmdline/commands/index.htm + import * as _7z from "node-7z"; +-import * as sevenBin from "7zip-bin"; +-import * as path from "path"; + import { exec } from "child_process"; + import joplin from "api"; +- +-export let pathTo7zip = sevenBin.path7za; +- +-export namespace sevenZip { +- export async function updateBinPath() { +- pathTo7zip = path.join( +- await joplin.plugins.installationDir(), +- "7zip-bin", +- pathTo7zip +- ); +- } ++const sevenBin = joplin.require("7zip-bin"); ++ ++ export let pathTo7zip = sevenBin.path7za; ++ ++ export namespace sevenZip { ++ export async function updateBinPath() { ++ // Not necessary with 7zip required from Joplin ++ // pathTo7zip = path.join( ++ // await joplin.plugins.installationDir(), ++ // "7zip-bin", ++ // pathTo7zip ++ // ); ++ } + + export async function setExecutionFlag() { + if (process.platform !== "win32") { +diff --git a/webpack.config.js b/webpack.config.js +index 34a1797..7b2a480 100644 +--- a/webpack.config.js ++++ b/webpack.config.js +@@ -200,15 +200,9 @@ const pluginConfig = { ...baseConfig, entry: './src/index.ts', + path: distDir, + }, + plugins: [ +- new CopyPlugin({ +- patterns: [ +- { +- from: '**/*', +- context: path.resolve(__dirname, 'node_modules','7zip-bin'), +- to: path.resolve(__dirname, 'dist/7zip-bin/'), +- }, +- ] +- }), ++ // Removed a CopyPlugin (added by Simple Backup, not necessary when using ++ // Joplin's built-in 7zip) ++ + new CopyPlugin({ + patterns: [ + { diff --git a/packages/default-plugins/pluginRepositories.json b/packages/default-plugins/pluginRepositories.json new file mode 100644 index 0000000000..687761963a --- /dev/null +++ b/packages/default-plugins/pluginRepositories.json @@ -0,0 +1,7 @@ +{ + "io.github.jackgruber.backup": { + "cloneUrl": "https://github.com/JackGruber/joplin-plugin-backup.git", + "branch": "master", + "commit": "021085cc37ed83a91a7950744e462782e27c04a6" + } +} diff --git a/packages/default-plugins/tsconfig.json b/packages/default-plugins/tsconfig.json new file mode 100644 index 0000000000..9b8814d1ed --- /dev/null +++ b/packages/default-plugins/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.json", + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "**/node_modules", + "plugin-sources/", + "plugin-base-repo/" + ] +} diff --git a/packages/default-plugins/utils/getPathToPatchFileFor.ts b/packages/default-plugins/utils/getPathToPatchFileFor.ts new file mode 100644 index 0000000000..936203f48f --- /dev/null +++ b/packages/default-plugins/utils/getPathToPatchFileFor.ts @@ -0,0 +1,9 @@ + +import { join, dirname } from 'path'; + +const getPathToPatchFileFor = (pluginName: string) => { + const rootDir = dirname(__dirname); + return join(rootDir, 'plugin-patches', `${pluginName}.diff`); +}; + +export default getPathToPatchFileFor; diff --git a/packages/default-plugins/utils/readRepositoryJson.ts b/packages/default-plugins/utils/readRepositoryJson.ts new file mode 100644 index 0000000000..01fa3f3831 --- /dev/null +++ b/packages/default-plugins/utils/readRepositoryJson.ts @@ -0,0 +1,37 @@ +import { readFile } from 'fs-extra'; + +export interface RepositoryData { + cloneUrl: string; + branch: string; + commit: string; +} + +export interface AllRepositoryData { + [pluginId: string]: RepositoryData; +} + +const readRepositoryJson = async (repositoryDataFilepath: string): Promise => { + const fileContent = await readFile(repositoryDataFilepath, 'utf8'); + const parsedJson = JSON.parse(fileContent); + + // Validate + for (const pluginId in parsedJson) { + if (typeof parsedJson[pluginId] !== 'object') { + throw new Error('pluginRepositories should map from plugin IDs to objects.'); + } + + const assertPropertyIsString = (propertyName: string) => { + if (typeof parsedJson[pluginId][propertyName] !== 'string') { + throw new Error(`Plugin ${pluginId} should have field '${propertyName}' of type string.`); + } + }; + + assertPropertyIsString('cloneUrl'); + assertPropertyIsString('branch'); + assertPropertyIsString('commit'); + } + + return parsedJson; +}; + +export default readRepositoryJson; diff --git a/packages/default-plugins/utils/waitForCliInput.ts b/packages/default-plugins/utils/waitForCliInput.ts new file mode 100644 index 0000000000..282b15a4b5 --- /dev/null +++ b/packages/default-plugins/utils/waitForCliInput.ts @@ -0,0 +1,23 @@ + +const readline = require('readline/promises'); + +/* eslint-disable no-console */ + +let readlineInterface: any = null; +const waitForCliInput = async () => { + readlineInterface ??= readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + if (process.stdin.isTTY) { + const green = '\x1b[92m'; + const reset = '\x1b[0m'; + await readlineInterface.question(`${green}[Press enter to continue]${reset}`); + + console.log('Continuing...'); + } else { + console.warn('Input is not from a TTY -- not waiting for input.'); + } +}; + +export default waitForCliInput; diff --git a/packages/lib/services/plugins/PluginService.ts b/packages/lib/services/plugins/PluginService.ts index ebf4742139..f6227b7f4f 100644 --- a/packages/lib/services/plugins/PluginService.ts +++ b/packages/lib/services/plugins/PluginService.ts @@ -33,7 +33,6 @@ export interface SettingAndValue { } export interface DefaultPluginSettings { - version: string; settings?: SettingAndValue; } diff --git a/packages/lib/services/plugins/defaultPlugins/defaultPluginsUtils.ts b/packages/lib/services/plugins/defaultPlugins/defaultPluginsUtils.ts index 37d86d1599..1e31428a0d 100644 --- a/packages/lib/services/plugins/defaultPlugins/defaultPluginsUtils.ts +++ b/packages/lib/services/plugins/defaultPlugins/defaultPluginsUtils.ts @@ -35,7 +35,10 @@ export async function installDefaultPlugins(service: PluginService, defaultPlugi const pluginId = pluginStat.path; // if pluginId is present in 'installedDefaultPlugins' array or it doesn't have default plugin ID, then we won't install it again as default plugin - if (installedPlugins.includes(pluginId) || !defaultPluginsId.includes(pluginId)) continue; + if (installedPlugins.includes(pluginId) || !defaultPluginsId.includes(pluginId)) { + logger.debug(`Skipping default plugin ${pluginId}, ${!defaultPluginsId.includes(pluginId) ? '(Not a default)' : ''}`); + continue; + } const defaultPluginPath: string = path.join(defaultPluginsDir, pluginId, 'plugin.jpl'); await service.installPlugin(defaultPluginPath, false); diff --git a/packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.ts b/packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.ts index 878463815a..6f36aec74d 100644 --- a/packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.ts +++ b/packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.ts @@ -4,14 +4,10 @@ import Setting from '../../../models/Setting'; const getDefaultPluginsInfo = (): DefaultPluginsInfo => { const defaultPlugins = { 'io.github.jackgruber.backup': { - version: '1.1.1', settings: { 'path': `${Setting.value('profileDir')}`, }, }, - 'plugin.calebjohn.rich-markdown': { - version: '0.8.3', - }, }; return defaultPlugins; }; diff --git a/packages/tools/bundleDefaultPlugins.test.ts b/packages/tools/bundleDefaultPlugins.test.ts deleted file mode 100644 index c3e3f1d48f..0000000000 --- a/packages/tools/bundleDefaultPlugins.test.ts +++ /dev/null @@ -1,206 +0,0 @@ - - - -import { join } from 'path'; -import { downloadPlugins, extractPlugins, localPluginsVersion } from './bundleDefaultPlugins'; -import { pathExists, readFile, remove } from 'fs-extra'; -import Setting from '@joplin/lib/models/Setting'; -import { createTempDir, supportDir } from '@joplin/lib/testing/test-utils'; -import { rootDir } from './tool-utils'; - -const fetch = require('node-fetch'); - -jest.mock('node-fetch', ()=>jest.fn()); - -const manifests = { - 'io.github.jackgruber.backup': { - 'manifest_version': 1, - 'id': 'io.github.jackgruber.backup', - 'app_min_version': '2.1.3', - 'version': '1.1.0', - 'name': 'Simple Backup', - 'description': 'Plugin to create manual and automatic backups.', - 'author': 'JackGruber', - 'homepage_url': 'https://github.com/JackGruber/joplin-plugin-backup/blob/master/README.md', - 'repository_url': 'https://github.com/JackGruber/joplin-plugin-backup', - 'keywords': [ - 'backup', - 'jex', - 'export', - 'zip', - '7zip', - 'encrypted', - ], - '_publish_hash': 'sha256:8d8c6a3bb92fafc587269aea58b623b05242d42c0766a05bbe25c3ba2bbdf8ee', - '_publish_commit': 'master:00ed52133c659e0f3ac1a55f70b776c42fca0a6d', - '_npm_package_name': 'joplin-plugin-backup', - }, - 'plugin.calebjohn.rich-markdown': { - 'manifest_version': 1, - 'id': 'plugin.calebjohn.rich-markdown', - 'app_min_version': '2.7', - 'version': '0.9.0', - 'name': 'Rich Markdown', - 'description': 'Helping you ditch the markdown viewer for good.', - 'author': 'Caleb John', - 'homepage_url': 'https://github.com/CalebJohn/joplin-rich-markdown#readme', - 'repository_url': 'https://github.com/CalebJohn/joplin-rich-markdown', - 'keywords': [ - 'editor', - 'visual', - ], - '_publish_hash': 'sha256:95337a3868aebdc9bf8c347a37460d0c2753b391ff51a0c72bdccdef9679705f', - '_publish_commit': 'main:af3493b6ca96c931327ab3bd04906faaed0c782c', - '_npm_package_name': 'joplin-plugin-rich-markdown', - }, - -}; - -const NPM_Response1 = JSON.stringify({ - '_id': 'joplin-plugin-rich-markdown', - 'name': 'joplin-plugin-rich-markdown', - 'versions': { - '0.8.2': { - 'name': 'joplin-plugin-rich-markdown', - 'version': '0.8.2', - 'description': 'A plugin that will finally allow you to ditch the markdown viewer, saving space and making your life easier.', - '_id': 'joplin-plugin-rich-markdown@0.1.0', - 'dist': { - 'tarball': 'no-link-here', - }, - }, - '0.9.0': { - 'name': 'joplin-plugin-rich-markdown', - 'version': '0.9.0', - 'dist': { - 'tarball': 'response-1-link', - }, - }, - }, -}); - -const NPM_Response2 = JSON.stringify({ - '_id': 'io.github.jackgruber.backup', - 'name': 'joplin-plugin-rich-markdown', - 'versions': { - '1.0.0': { - 'name': 'joplin-plugin-rich-markdown', - 'version': '1.0.0', - 'description': 'A plugin that will finally allow you to ditch the markdown viewer, saving space and making your life easier.', - '_id': 'joplin-plugin-rich-markdown@0.1.0', - 'dist': { - 'tarball': 'no-link-here', - }, - }, - '1.1.0': { - 'name': 'joplin-plugin-rich-markdown', - 'version': '1.1.0', - 'dist': { - 'tarball': 'response-2-link', - }, - }, - }, -}); - -async function mockPluginData() { - const filePath = join(__dirname, '..', 'app-cli', 'tests', 'services', 'plugins', 'mockData', 'mockPlugin.tgz'); - const tgzData = await readFile(filePath, 'utf8'); - return tgzData; -} - -describe('bundleDefaultPlugins', () => { - - const testDefaultPluginsInfo = { - 'plugin.calebjohn.rich-markdown': { - version: '0.9.0', - }, - 'io.github.jackgruber.backup': { - version: '1.1.0', - settings: { - 'path': `${Setting.value('profileDir')}`, - }, - }, - }; - - it('should get local plugin versions', async () => { - const manifestsPath = join(supportDir, 'pluginRepo', 'plugins'); - const testDefaultPluginsInfo = { - 'joplin.plugin.ambrt.backlinksToNote': { version: '1.0.4' }, - 'org.joplinapp.plugins.ToggleSidebars': { version: '1.0.2' }, - }; - const localPluginsVersions = await localPluginsVersion(manifestsPath, testDefaultPluginsInfo); - - expect(localPluginsVersions['joplin.plugin.ambrt.backlinksToNote']).toBe('1.0.4'); - expect(localPluginsVersions['org.joplinapp.plugins.ToggleSidebars']).toBe('1.0.2'); - }); - - it('should download plugins folder from GitHub with no initial plugins', async () => { - - const testCases = [ - { - localVersions: { 'io.github.jackgruber.backup': '0.0.0', 'plugin.calebjohn.rich-markdown': '0.0.0' }, - downloadedPlugin1: 'joplin-plugin-rich-markdown-0.9.0.tgz', - downloadedPlugin2: 'joplin-plugin-backup-1.1.0.tgz', - numberOfCalls: 4, - calledWith: ['https://registry.npmjs.org/joplin-plugin-rich-markdown', 'response-1-link', 'https://registry.npmjs.org/joplin-plugin-backup', 'response-2-link'], - }, - { - localVersions: { 'io.github.jackgruber.backup': '1.1.0', 'plugin.calebjohn.rich-markdown': '0.0.0' }, - downloadedPlugin1: 'joplin-plugin-rich-markdown-0.9.0.tgz', - downloadedPlugin2: undefined, - numberOfCalls: 2, - calledWith: ['https://registry.npmjs.org/joplin-plugin-rich-markdown', 'response-1-link'], - }, - { - localVersions: { 'io.github.jackgruber.backup': '1.1.0', 'plugin.calebjohn.rich-markdown': '0.9.0' }, - downloadedPlugin1: undefined, - downloadedPlugin2: undefined, - numberOfCalls: 0, - calledWith: [], - }, - ]; - - const tgzData = await mockPluginData(); - - const mockFetch = fetch as jest.MockedFunction; - - for (const testCase of testCases) { - - mockFetch.mockResolvedValueOnce({ text: () => Promise.resolve(NPM_Response1), ok: true }) - .mockResolvedValueOnce({ buffer: () => Promise.resolve(tgzData), ok: true }) - .mockResolvedValueOnce({ text: () => Promise.resolve(NPM_Response2), ok: true }) - .mockResolvedValueOnce({ buffer: () => Promise.resolve(tgzData), ok: true }); - - const downloadedPlugins = await downloadPlugins(testCase.localVersions, testDefaultPluginsInfo, manifests); - - expect(downloadedPlugins[Object.keys(testDefaultPluginsInfo)[0]]).toBe(testCase.downloadedPlugin1); - expect(downloadedPlugins[Object.keys(testDefaultPluginsInfo)[1]]).toBe(testCase.downloadedPlugin2); - - expect(mockFetch).toHaveBeenCalledTimes(testCase.numberOfCalls); - - // eslint-disable-next-line github/array-foreach -- Old code before rule was applied - testCase.calledWith.forEach((callValue, index) => expect(mockFetch).toHaveBeenNthCalledWith(index + 1, callValue)); - - jest.clearAllMocks(); - } - - await remove(`${rootDir}/packages/tools/joplin-plugin-backup-1.1.0.tgz`); - await remove(`${rootDir}/packages/tools/joplin-plugin-rich-markdown-0.9.0.tgz`); - }); - - it('should extract plugins files', async () => { - - const downloadedPluginsNames = { 'plugin.calebjohn.rich-markdown': 'mockPlugin.tgz' }; - - const filePath = join(__dirname, '..', 'app-cli', 'tests', 'services', 'plugins', 'mockData'); - const tempDir = await createTempDir(); - - await extractPlugins(filePath, tempDir, downloadedPluginsNames); - - expect(await pathExists(join(tempDir, 'plugin.calebjohn.rich-markdown', 'plugin.jpl'))).toBe(true); - expect(await pathExists(join(tempDir, 'plugin.calebjohn.rich-markdown', 'manifest.json'))).toBe(true); - - await remove(tempDir); - }); - -}); diff --git a/packages/tools/bundleDefaultPlugins.ts b/packages/tools/bundleDefaultPlugins.ts deleted file mode 100644 index 66e4d7baed..0000000000 --- a/packages/tools/bundleDefaultPlugins.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { join } from 'path'; -import { pathExists, mkdir, readFile, move, remove, writeFile } from 'fs-extra'; -import { DefaultPluginsInfo } from '@joplin/lib/services/plugins/PluginService'; -import getDefaultPluginsInfo from '@joplin/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo'; -import { execCommand } from '@joplin/utils'; -const fetch = require('node-fetch'); - -interface PluginAndVersion { - [pluginId: string]: string; -} - -interface PluginIdAndName { - [pluginId: string]: string; -} - -export const localPluginsVersion = async (defaultPluginDir: string, defaultPluginsInfo: DefaultPluginsInfo): Promise => { - if (!await pathExists(join(defaultPluginDir))) await mkdir(defaultPluginDir); - const localPluginsVersions: PluginAndVersion = {}; - - for (const pluginId of Object.keys(defaultPluginsInfo)) { - - if (!await pathExists(join(defaultPluginDir, pluginId))) { - localPluginsVersions[pluginId] = '0.0.0'; - continue; - } - const data = await readFile(`${defaultPluginDir}/${pluginId}/manifest.json`, 'utf8'); - const manifest = JSON.parse(data); - localPluginsVersions[pluginId] = manifest.version; - } - return localPluginsVersions; -}; - -async function downloadFile(url: string, outputPath: string) { - const response = await fetch(url); - if (!response.ok) { - const responseText = await response.text(); - throw new Error(`Cannot download file from ${url} : ${responseText.substr(0, 500)}`); - } - await writeFile(outputPath, await response.buffer()); -} - -export async function extractPlugins(currentDir: string, defaultPluginDir: string, downloadedPluginsNames: PluginIdAndName): Promise { - for (const pluginId of Object.keys(downloadedPluginsNames)) { - await execCommand(`tar xzf ${currentDir}/${downloadedPluginsNames[pluginId]}`, { quiet: true }); - await move(`package/publish/${pluginId}.jpl`, `${defaultPluginDir}/${pluginId}/plugin.jpl`, { overwrite: true }); - await move(`package/publish/${pluginId}.json`, `${defaultPluginDir}/${pluginId}/manifest.json`, { overwrite: true }); - await remove(`${downloadedPluginsNames[pluginId]}`); - await remove('package'); - } -} - -export const downloadPlugins = async (localPluginsVersions: PluginAndVersion, defaultPluginsInfo: DefaultPluginsInfo, manifests: any): Promise => { - - const downloadedPluginsNames: PluginIdAndName = {}; - for (const pluginId of Object.keys(defaultPluginsInfo)) { - if (localPluginsVersions[pluginId] === defaultPluginsInfo[pluginId].version) continue; - const response = await fetch(`https://registry.npmjs.org/${manifests[pluginId]._npm_package_name}`); - - if (!response.ok) { - const responseText = await response.text(); - throw new Error(`Cannot fetch ${manifests[pluginId]._npm_package_name} release info from NPM : ${responseText.substr(0, 500)}`); - } - const releaseText = await response.text(); - const release = JSON.parse(releaseText); - - const pluginUrl = release.versions[defaultPluginsInfo[pluginId].version].dist.tarball; - - const pluginName = `${manifests[pluginId]._npm_package_name}-${defaultPluginsInfo[pluginId].version}.tgz`; - await downloadFile(pluginUrl, pluginName); - - downloadedPluginsNames[pluginId] = pluginName; - } - return downloadedPluginsNames; -}; - -async function start(): Promise { - const defaultPluginDir = join(__dirname, '..', '..', 'packages', 'app-desktop', 'build', 'defaultPlugins'); - const defaultPluginsInfo = getDefaultPluginsInfo(); - - const manifestData = await fetch('https://raw.githubusercontent.com/joplin/plugins/master/manifests.json'); - const manifests = JSON.parse(await manifestData.text()); - if (!manifests) throw new Error('Invalid or missing JSON'); - - const localPluginsVersions = await localPluginsVersion(defaultPluginDir, defaultPluginsInfo); - const downloadedPluginNames: PluginIdAndName = await downloadPlugins(localPluginsVersions, defaultPluginsInfo, manifests); - await extractPlugins(__dirname, defaultPluginDir, downloadedPluginNames); -} - -if (require.main === module) { -// eslint-disable-next-line promise/prefer-await-to-then -- Old code before rule was applied - start().catch((error) => { - console.error('Fatal error'); - console.error(error); - process.exit(1); - }); -} diff --git a/packages/tools/gulp/tasks/updateIgnoredTypeScriptBuild.js b/packages/tools/gulp/tasks/updateIgnoredTypeScriptBuild.js index 60e776c1f6..05d110a723 100644 --- a/packages/tools/gulp/tasks/updateIgnoredTypeScriptBuild.js +++ b/packages/tools/gulp/tasks/updateIgnoredTypeScriptBuild.js @@ -31,6 +31,7 @@ module.exports = { 'packages/app-desktop/dist/**', 'packages/app-mobile/android/**', 'packages/app-mobile/ios/**', + 'packages/default-plugins/plugin-sources/**', 'packages/fork-sax/**', 'packages/lib/plugin_types/**', 'packages/server/**', diff --git a/packages/tools/setupNewRelease.ts b/packages/tools/setupNewRelease.ts index 529b9101ab..024303f909 100644 --- a/packages/tools/setupNewRelease.ts +++ b/packages/tools/setupNewRelease.ts @@ -140,6 +140,7 @@ async function main() { await updatePackageVersion(`${rootDir}/packages/server/package.json`, majorMinorVersion, options); await updatePackageVersion(`${rootDir}/packages/tools/package.json`, majorMinorVersion, options); await updatePackageVersion(`${rootDir}/packages/utils/package.json`, majorMinorVersion, options); + await updatePackageVersion(`${rootDir}/packages/default-plugins/package.json`, majorMinorVersion, options); await updatePackageVersion(`${rootDir}/packages/editor/package.json`, majorMinorVersion, options); if (options.updateVersion) { diff --git a/packages/utils/package.json b/packages/utils/package.json index e1c40ce70d..4d98d0dd3d 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -19,6 +19,7 @@ }, "scripts": { "tsc": "tsc --project tsconfig.json", + "build": "yarn run tsc", "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json", "test": "jest --verbose=false", "test-ci": "yarn test" diff --git a/readme/dev/spec/default_plugins.md b/readme/dev/spec/default_plugins.md index a688907d6c..e1fce59ec7 100644 --- a/readme/dev/spec/default_plugins.md +++ b/readme/dev/spec/default_plugins.md @@ -9,7 +9,6 @@ To add a new default plugin for desktop: ``` const defaultPlugins = { 'samplePluginId': { - version: '1.0.0', settings: { 'settingName1': 'setting-value1', 'settingName2': 'setting-value2', @@ -18,13 +17,37 @@ const defaultPlugins = { }; ``` +After this, add the commit, branch, and clone URL to be build from to `pluginRepositories.json`. + +For example, +```json +{ + "plugin.id.here": { + "cloneUrl": "https://example.com/plugin-repo/plugin-repo-here.git", + "branch": "main", + "commit": "840d2e84b70adf6de961e167dcd27ddad088b286" + } +} +``` + +## Patching the plugin + +Some plugins need patching. To create or update a plugin's patch, run the `patch` command in the `packages/default-plugins/` directory. + +For example, +```shell +$ cd packages/default-plugins +$ yarn run patch plugin.id.here +``` + +The script will create a temporary directory in which changes can be made. Do not stage the changes that should appear in the patch. + ## Bundling of default plugins -Script for bundling default plugins is present in [bundleDefaultPlugins.ts](https://github.com/laurent22/joplin/blob/eb7083d7888433ff6ef76ccfb7fb87ba951d513f/packages/tools/bundleDefaultPlugins.ts) +Scripts for bundling default plugins are present in `packages/default-plugins/`. -Every time a new desktop release is being built, we compare the local default plugins version with pinned plugin version mentioned in [desktopDefaultPluginsInfo.ts](https://github.com/laurent22/joplin/blob/eb7083d7888433ff6ef76ccfb7fb87ba951d513f/packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.ts) +These are run by the `app-desktop` package on a full `build` (e.g. on `postinstall`). -If there is a newer version available, we will pull the `tgz` file of plugin from NPM registry and extract it. We will then move `manifest.json` and `plugin.jpl` to the build folder of desktop. ## Installing of default plugins diff --git a/yarn.lock b/yarn.lock index 6f9811e76c..b8a9659a01 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5,24 +5,10 @@ __metadata: version: 6 cacheKey: 8 -"7zip-bin-linux@npm:^1.0.1": - version: 1.3.1 - resolution: "7zip-bin-linux@npm:1.3.1" - conditions: os=linux - languageName: node - linkType: hard - -"7zip-bin-mac@npm:^1.0.1": - version: 1.0.1 - resolution: "7zip-bin-mac@npm:1.0.1" - conditions: os=darwin - languageName: node - linkType: hard - -"7zip-bin-win@npm:^2.1.1": - version: 2.2.0 - resolution: "7zip-bin-win@npm:2.2.0" - conditions: os=win32 +"7zip-bin@npm:5.2.0": + version: 5.2.0 + resolution: "7zip-bin@npm:5.2.0" + checksum: 85d3102275342f1f4ba7d17e778e526dee3dbec0f57d29be7afaa6e3c26687d40a6eccf520e9140143f85a51f3353f6b545f760eff3f776c6ffb30dc5252fb7c languageName: node linkType: hard @@ -6411,14 +6397,13 @@ __metadata: version: 0.0.0-use.local resolution: "@joplin/app-desktop@workspace:packages/app-desktop" dependencies: - 7zip-bin-linux: ^1.0.1 - 7zip-bin-mac: ^1.0.1 - 7zip-bin-win: ^2.1.1 + 7zip-bin: 5.2.0 "@electron/notarize": 2.1.0 "@electron/rebuild": 3.3.0 "@electron/remote": 2.0.12 "@fortawesome/fontawesome-free": 5.15.4 "@joeattardi/emoji-button": 4.6.4 + "@joplin/default-plugins": ~2.13 "@joplin/editor": ~2.13 "@joplin/lib": ~2.13 "@joplin/renderer": ~2.13 @@ -6478,14 +6463,8 @@ __metadata: styled-system: 5.1.5 taboverride: 4.0.3 tinymce: 5.10.6 + ts-node: 10.9.1 typescript: 5.2.2 - dependenciesMeta: - 7zip-bin-linux: - optional: true - 7zip-bin-mac: - optional: true - 7zip-bin-win: - optional: true languageName: unknown linkType: soft @@ -6594,6 +6573,19 @@ __metadata: languageName: unknown linkType: soft +"@joplin/default-plugins@workspace:packages/default-plugins, @joplin/default-plugins@~2.13": + version: 0.0.0-use.local + resolution: "@joplin/default-plugins@workspace:packages/default-plugins" + dependencies: + "@joplin/utils": ~2.13 + "@types/yargs": 17.0.31 + fs-extra: 11.1.1 + ts-node: 10.9.1 + typescript: 5.2.2 + yargs: 17.7.2 + languageName: unknown + linkType: soft + "@joplin/doc-builder@workspace:packages/doc-builder": version: 0.0.0-use.local resolution: "@joplin/doc-builder@workspace:packages/doc-builder"