Desktop: Resolves #7934: Add Simple Backup as a default plugin (#9360)

.eslintignore

@@ -59,6 +59,8 @@ packages/app-mobile/locales
 packages/app-mobile/node_modules
 packages/app-mobile/pluginAssets/
 packages/fork-*
+packages/default-plugins/plugin-base-repo/
+packages/default-plugins/plugin-sources/
 packages/htmlpack/dist/
 packages/lib/assets/
 packages/lib/lib/lib.js
@@ -380,10 +382,12 @@ packages/app-desktop/gui/style/StyledTextInput.js
 packages/app-desktop/gui/utils/NoteListUtils.js
 packages/app-desktop/gui/utils/convertToScreenCoordinates.js
 packages/app-desktop/gui/utils/loadScript.js
+packages/app-desktop/gulpfile.js
 packages/app-desktop/integration-tests/main.spec.js
 packages/app-desktop/integration-tests/models/MainScreen.js
 packages/app-desktop/integration-tests/models/NoteEditorScreen.js
 packages/app-desktop/integration-tests/models/SettingsScreen.js
+packages/app-desktop/integration-tests/simpleBackup.spec.js
 packages/app-desktop/integration-tests/util/activateMainMenuItem.js
 packages/app-desktop/integration-tests/util/createStartupArgs.js
 packages/app-desktop/integration-tests/util/firstNonDevToolsWindow.js
@@ -415,6 +419,7 @@ packages/app-desktop/services/sortOrder/PerFolderSortOrderService.js
 packages/app-desktop/services/sortOrder/notesSortOrderUtils.test.js
 packages/app-desktop/services/sortOrder/notesSortOrderUtils.js
 packages/app-desktop/services/spellChecker/SpellCheckerServiceDriverNative.js
+packages/app-desktop/tools/copy7Zip.js
 packages/app-desktop/tools/notarizeMacApp.js
 packages/app-desktop/tools/renameReleaseAssets.js
 packages/app-desktop/utils/checkForUpdatesUtils.test.js
@@ -530,6 +535,13 @@ packages/app-mobile/utils/fs-driver/runOnDeviceTests.js
 packages/app-mobile/utils/setupNotifications.js
 packages/app-mobile/utils/shareHandler.js
 packages/app-mobile/utils/types.js
+packages/default-plugins/build.js
+packages/default-plugins/buildDefaultPlugins.js
+packages/default-plugins/commands/buildAll.js
+packages/default-plugins/commands/editPatch.js
+packages/default-plugins/utils/getPathToPatchFileFor.js
+packages/default-plugins/utils/readRepositoryJson.js
+packages/default-plugins/utils/waitForCliInput.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.test.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.js
 packages/editor/CodeMirror/CodeMirror5Emulation/Decorator.js
@@ -1013,8 +1025,6 @@ packages/tools/build-translation.js
 packages/tools/build-welcome.js
 packages/tools/buildServerDocker.test.js
 packages/tools/buildServerDocker.js
-packages/tools/bundleDefaultPlugins.test.js
-packages/tools/bundleDefaultPlugins.js
 packages/tools/checkIgnoredFiles.js
 packages/tools/checkLibPaths.test.js
 packages/tools/checkLibPaths.js

.gitignore

@@ -362,10 +362,12 @@ packages/app-desktop/gui/style/StyledTextInput.js
 packages/app-desktop/gui/utils/NoteListUtils.js
 packages/app-desktop/gui/utils/convertToScreenCoordinates.js
 packages/app-desktop/gui/utils/loadScript.js
+packages/app-desktop/gulpfile.js
 packages/app-desktop/integration-tests/main.spec.js
 packages/app-desktop/integration-tests/models/MainScreen.js
 packages/app-desktop/integration-tests/models/NoteEditorScreen.js
 packages/app-desktop/integration-tests/models/SettingsScreen.js
+packages/app-desktop/integration-tests/simpleBackup.spec.js
 packages/app-desktop/integration-tests/util/activateMainMenuItem.js
 packages/app-desktop/integration-tests/util/createStartupArgs.js
 packages/app-desktop/integration-tests/util/firstNonDevToolsWindow.js
@@ -397,6 +399,7 @@ packages/app-desktop/services/sortOrder/PerFolderSortOrderService.js
 packages/app-desktop/services/sortOrder/notesSortOrderUtils.test.js
 packages/app-desktop/services/sortOrder/notesSortOrderUtils.js
 packages/app-desktop/services/spellChecker/SpellCheckerServiceDriverNative.js
+packages/app-desktop/tools/copy7Zip.js
 packages/app-desktop/tools/notarizeMacApp.js
 packages/app-desktop/tools/renameReleaseAssets.js
 packages/app-desktop/utils/checkForUpdatesUtils.test.js
@@ -512,6 +515,13 @@ packages/app-mobile/utils/fs-driver/runOnDeviceTests.js
 packages/app-mobile/utils/setupNotifications.js
 packages/app-mobile/utils/shareHandler.js
 packages/app-mobile/utils/types.js
+packages/default-plugins/build.js
+packages/default-plugins/buildDefaultPlugins.js
+packages/default-plugins/commands/buildAll.js
+packages/default-plugins/commands/editPatch.js
+packages/default-plugins/utils/getPathToPatchFileFor.js
+packages/default-plugins/utils/readRepositoryJson.js
+packages/default-plugins/utils/waitForCliInput.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.test.js
 packages/editor/CodeMirror/CodeMirror5Emulation/CodeMirror5Emulation.js
 packages/editor/CodeMirror/CodeMirror5Emulation/Decorator.js
@@ -995,8 +1005,6 @@ packages/tools/build-translation.js
 packages/tools/build-welcome.js
 packages/tools/buildServerDocker.test.js
 packages/tools/buildServerDocker.js
-packages/tools/bundleDefaultPlugins.test.js
-packages/tools/bundleDefaultPlugins.js
 packages/tools/checkIgnoredFiles.js
 packages/tools/checkLibPaths.test.js
 packages/tools/checkLibPaths.js

.npmpackagejsonlintignore

@@ -1,3 +1,5 @@
 packages/app-clipper/popup/
 packages/app-cli/tests/support/plugins/
-packages/doc-builder/
+packages/doc-builder/
+packages/default-plugins/plugin-base-repo/
+packages/default-plugins/plugin-sources/

packages/app-cli/tests/services/plugins/defaultPluginsUtils.ts

@@ -189,13 +189,11 @@ describe('defaultPluginsUtils', () => {
 
 		const defaultPluginsInfo: DefaultPluginsInfo = {
 			'io.github.jackgruber.backup': {
-				version: '1.0.2',
 				settings: {
 					'path': `${Setting.value('profileDir')}`,
 				},
 			},
 			'plugin.calebjohn.rich-markdown': {
-				version: '0.8.3',
 			},
 		};
 
@@ -245,14 +243,12 @@ describe('defaultPluginsUtils', () => {
 
 		const defaultPluginsInfo: DefaultPluginsInfo = {
 			'io.github.jackgruber.backup': {
-				version: '1.0.2',
 				settings: {
 					'path': `${Setting.value('profileDir')}`,
 					'missing-key1': 'someValue',
 				},
 			},
 			'plugin.calebjohn.rich-markdown': {
-				version: '0.8.3',
 				settings: {
 					'missing-key2': 'someValue',
 				},

packages/app-desktop/.gitignore

@@ -18,3 +18,6 @@ test-results/
 playwright-report/
 playwright/.cache/
 integration-tests/test-profile/
+build/defaultPlugins/
+build/7zip/7za
+build/7zip/7za.exe

packages/app-desktop/app.ts

@@ -29,7 +29,7 @@ import { reg } from '@joplin/lib/registry';
 const packageInfo: PackageInfo = require('./packageInfo.js');
 import DecryptionWorker from '@joplin/lib/services/DecryptionWorker';
 import ClipperServer from '@joplin/lib/ClipperServer';
-const { webFrame } = require('electron');
+import { ipcRenderer, webFrame } from 'electron';
 const Menu = bridge().Menu;
 const PluginManager = require('@joplin/lib/services/PluginManager');
 import RevisionService from '@joplin/lib/services/RevisionService';
@@ -333,6 +333,11 @@ class Application extends BaseApplication {
 					type: 'STARTUP_PLUGINS_LOADED',
 					value: true,
 				});
+
+				// Sends an event to the main process -- this is used by the Playwright
+				// tests to wait for plugins to load.
+				ipcRenderer.send('startup-plugins-loaded');
+
 				setSettingsForDefaultPlugins(getDefaultPluginsInfo());
 			}
 		}, 500);

packages/app-desktop/build/7zip/license.txt

@@ -0,0 +1,90 @@
+7-Zip
+~~~~~
+License for use and distribution
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+7-Zip Copyright (C) 1999-2023 Igor Pavlov.
+
+The licenses for files are:
+
+1) 7z.dll:
+		- The "GNU LGPL" as main license for most of the code
+		- The "GNU LGPL" with "unRAR license restriction" for some code
+		- The "BSD 3-clause License" for some code
+2) All other files: the "GNU LGPL".
+
+Redistributions in binary form must reproduce related license information from this file.
+
+Note:
+You can use 7-Zip on any computer, including a computer in a commercial
+organization. You don't need to register or pay for 7-Zip.
+
+
+GNU LGPL information
+--------------------
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You can receive a copy of the GNU Lesser General Public License from
+http://www.gnu.org/
+
+
+
+
+BSD 3-clause License
+--------------------
+
+The "BSD 3-clause License" is used for the code in 7z.dll that implements LZFSE data decompression.
+That code was derived from the code in the "LZFSE compression library" developed by Apple Inc,
+that also uses the "BSD 3-clause License":
+
+----
+Copyright (c) 2015-2016, Apple Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1.  Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2.  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
+	in the documentation and/or other materials provided with the distribution.
+
+3.  Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
+	from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+----
+
+
+
+
+unRAR license restriction
+-------------------------
+
+The decompression engine for RAR archives was developed using source
+code of unRAR program.
+All copyrights to original unRAR code are owned by Alexander Roshal.
+
+The license for original unRAR code has the following restriction:
+
+	The unRAR sources cannot be used to re-create the RAR compression algorithm,
+	which is proprietary. Distribution of modified unRAR sources in separate form
+	or as a part of other software is permitted, provided that it is clearly
+	stated in the documentation and source comments that the code may
+	not be used to develop a RAR (WinRAR) compatible archiver.
+
+
+--
+Igor Pavlov

packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginBox.tsx

@@ -60,7 +60,7 @@ const CellRoot = styled.div<{ isCompatible: boolean }>`
 	box-sizing: border-box;
 	background-color: ${props => props.theme.backgroundColor};
 	flex-direction: column;
-	align-items: flex-start;
+	align-items: stretch;
 	padding: 15px;
 	border: 1px solid ${props => props.theme.dividerColor};
 	border-radius: 6px;
@@ -96,12 +96,15 @@ const NeedUpgradeMessage = styled.span`
 	font-size: ${props => props.theme.fontSize}px;
 `;
 
-const DevModeLabel = styled.div`
+const BoxedLabel = styled.div`
 	border: 1px solid ${props => props.theme.color};
 	border-radius: 4px;
 	padding: 4px 6px;
 	font-size: ${props => props.theme.fontSize * 0.75}px;
 	color: ${props => props.theme.color};
+	flex-grow: 0;
+	height: min-content;
+	margin-top: auto;
 `;
 
 const StyledNameAndVersion = styled.div<{ mb: any }>`
@@ -170,7 +173,7 @@ export default function(props: Props) {
 		if (!props.onToggle) return null;
 
 		if (item.devMode) {
-			return <DevModeLabel>DEV</DevModeLabel>;
+			return <BoxedLabel>DEV</BoxedLabel>;
 		}
 
 		return <ToggleButton
@@ -217,6 +220,17 @@ export default function(props: Props) {
 		/>;
 	}
 
+	const renderDefaultPluginLabel = () => {
+		// Built-in plugins can only be disabled
+		if (item.manifest._built_in) {
+			return (
+				<BoxedLabel>{_('Built in')}</BoxedLabel>
+			);
+		}
+
+		return null;
+	};
+
 	function renderFooter() {
 		if (item.devMode) return null;
 
@@ -236,6 +250,7 @@ export default function(props: Props) {
 				{renderInstallButton()}
 				{renderUpdateButton()}
 				<div style={{ display: 'flex', flex: 1 }}/>
+				{renderDefaultPluginLabel()}
 			</CellFooter>
 		);
 	}

packages/app-desktop/gui/ConfigScreen/controls/plugins/PluginsStates.tsx

@@ -147,11 +147,18 @@ export default function(props: Props) {
 		let cancelled = false;
 
 		async function fetchPluginIds() {
-			const pluginIds = await repoApi().canBeUpdatedPlugins(pluginItems.map(p => p.manifest), pluginService.appVersion);
+			// Built-in plugins can't be updated from the main repoApi
+			const nonDefaultPlugins = pluginItems
+				.map(p => p.manifest)
+				.filter(manifest => !manifest._built_in);
+
+			const pluginIds = await repoApi().canBeUpdatedPlugins(nonDefaultPlugins, pluginService.appVersion);
 			if (cancelled) return;
+
 			const conv: Record<string, boolean> = {};
-			// eslint-disable-next-line github/array-foreach -- Old code before rule was applied
+			for (const id of pluginIds) {
-			pluginIds.forEach(id => conv[id] = true);
+				conv[id] = true;
+			}
 			setCanBeUpdatedPluginIds(conv);
 		}
 
@@ -281,10 +288,17 @@ export default function(props: Props) {
 				</UserPluginsRoot>
 			);
 		} else {
+			const nonDefaultPlugins = pluginItems.filter(item => !item.manifest._built_in);
+			const defaultPlugins = pluginItems.filter(item => item.manifest._built_in);
 			return (
-				<UserPluginsRoot>
+				<>
-					{renderCells(pluginItems)}
+					<UserPluginsRoot>
-				</UserPluginsRoot>
+						{renderCells(nonDefaultPlugins)}
+					</UserPluginsRoot>
+					<UserPluginsRoot>
+						{renderCells(defaultPlugins)}
+					</UserPluginsRoot>
+				</>
 			);
 		}
 	}

packages/app-desktop/gulpfile.ts

@@ -2,6 +2,8 @@ const gulp = require('gulp');
 const utils = require('@joplin/tools/gulp/utils');
 const compileSass = require('@joplin/tools/compileSass');
 const compilePackageInfo = require('@joplin/tools/compilePackageInfo');
+import buildDefaultPlugins from '@joplin/default-plugins/commands/buildAll';
+import copy7Zip from './tools/copy7Zip';
 
 const tasks = {
 	compileScripts: {
@@ -24,6 +26,17 @@ const tasks = {
 	electronBuilder: {
 		fn: require('./tools/electronBuilder.js'),
 	},
+	copyDefaultPluginsAssets: {
+		fn: async () => {
+			await copy7Zip();
+		},
+	},
+	buildDefaultPlugins: {
+		fn: async () => {
+			const outputDir = `${__dirname}/build/defaultPlugins/`;
+			await buildDefaultPlugins(outputDir);
+		},
+	},
 	tsc: require('@joplin/tools/gulp/tasks/tsc'),
 	updateIgnoredTypeScriptBuild: require('@joplin/tools/gulp/tasks/updateIgnoredTypeScriptBuild'),
 	buildCommandIndex: require('@joplin/tools/gulp/tasks/buildCommandIndex'),
@@ -39,7 +52,7 @@ const tasks = {
 
 utils.registerGulpTasks(gulp, tasks);
 
-const buildParallel = [
+const buildBeforeStartParallel = [
 	'compileScripts',
 	'compilePackageInfo',
 	'copyPluginAssets',
@@ -49,4 +62,12 @@ const buildParallel = [
 	'compileSass',
 ];
 
-gulp.task('build', gulp.parallel(...buildParallel));
+gulp.task('before-start', gulp.parallel(...buildBeforeStartParallel));
+
+const buildAllSequential = [
+	'before-start',
+	'copyDefaultPluginsAssets',
+	'buildDefaultPlugins',
+];
+
+gulp.task('build', gulp.series(buildAllSequential));

packages/app-desktop/integration-tests/main.spec.ts

@@ -1,6 +1,5 @@
 import { test, expect } from './util/test';
 import MainScreen from './models/MainScreen';
-import activateMainMenuItem from './util/activateMainMenuItem';
 import SettingsScreen from './models/SettingsScreen';
 import { _electron as electron } from '@playwright/test';
 import { writeFile } from 'fs-extra';
@@ -91,10 +90,7 @@ test.describe('main', () => {
 		// Sort order buttons should be visible by default
 		await expect(mainScreen.noteListContainer.locator('[title^="Toggle sort order"]')).toBeVisible();
 
-		// Open settings (check both labels so that this works on MacOS)
+		await mainScreen.openSettings(electronApp);
-		expect(
-			await activateMainMenuItem(electronApp, 'Preferences...') || await activateMainMenuItem(electronApp, 'Options'),
-		).toBe(true);
 
 		// Should be on the settings screen
 		const settingsScreen = new SettingsScreen(mainWindow);

packages/app-desktop/integration-tests/models/MainScreen.ts

@@ -1,5 +1,6 @@
-import { Page, Locator } from '@playwright/test';
+import { Page, Locator, ElectronApplication } from '@playwright/test';
 import NoteEditorScreen from './NoteEditorScreen';
+import activateMainMenuItem from '../util/activateMainMenuItem';
 
 export default class MainScreen {
 	public readonly newNoteButton: Locator;
@@ -33,4 +34,14 @@ export default class MainScreen {
 
 		return this.noteEditor;
 	}
+
+	public async openSettings(electronApp: ElectronApplication) {
+		// Check both labels so this works on MacOS
+		const openedWithPreferences = await activateMainMenuItem(electronApp, 'Preferences...');
+		const openedWithOptions = await activateMainMenuItem(electronApp, 'Options');
+
+		if (!openedWithOptions && !openedWithPreferences) {
+			throw new Error('Unable to find settings menu item in application menus.');
+		}
+	}
 }

packages/app-desktop/integration-tests/models/SettingsScreen.ts

@@ -5,9 +5,13 @@ export default class SettingsScreen {
 	public readonly okayButton: Locator;
 	public readonly appearanceTabButton: Locator;
 
-	public constructor(page: Page) {
+	public constructor(private page: Page) {
 		this.okayButton = page.locator('button', { hasText: 'OK' });
-		this.appearanceTabButton = page.getByText('Appearance');
+		this.appearanceTabButton = this.getTabLocator('Appearance');
+	}
+
+	public getTabLocator(tabName: string) {
+		return this.page.locator('a[role="tab"] > span', { hasText: tabName });
 	}
 
 	public async waitFor() {

packages/app-desktop/integration-tests/simpleBackup.spec.ts

@@ -0,0 +1,41 @@
+import { test, expect } from './util/test';
+import MainScreen from './models/MainScreen';
+import SettingsScreen from './models/SettingsScreen';
+import activateMainMenuItem from './util/activateMainMenuItem';
+
+test.describe('simpleBackup', () => {
+	test('should have a section in settings', async ({ electronApp, startupPluginsLoaded, mainWindow }) => {
+		await startupPluginsLoaded;
+
+		const mainScreen = new MainScreen(mainWindow);
+		await mainScreen.waitFor();
+
+		// Open settings (check both labels so that this works on MacOS)
+		await mainScreen.openSettings(electronApp);
+
+		// Should be on the settings screen
+		const settingsScreen = new SettingsScreen(mainWindow);
+		await settingsScreen.waitFor();
+
+		const backupTab = settingsScreen.getTabLocator('Backup');
+		await backupTab.waitFor();
+	});
+
+	test('should be possible to create a backup', async ({ electronApp, startupPluginsLoaded, mainWindow }) => {
+		await startupPluginsLoaded;
+
+		const mainScreen = new MainScreen(mainWindow);
+		await mainScreen.waitFor();
+
+		// Backups should work
+		expect(await activateMainMenuItem(electronApp, 'Create backup')).toBe(true);
+
+		const successDialog = mainWindow.locator('iframe[id$=backup-backupDialog]');
+		await successDialog.waitFor();
+
+		// Should report success
+		const dialogContentLocator = successDialog.frameLocator(':scope');
+		await dialogContentLocator.getByText('Backup completed').waitFor();
+	});
+});
+

packages/app-desktop/integration-tests/util/test.ts

@@ -10,6 +10,7 @@ import firstNonDevToolsWindow from './firstNonDevToolsWindow';
 type JoplinFixtures = {
 	profileDirectory: string;
 	electronApp: ElectronApplication;
+	startupPluginsLoaded: Promise<void>;
 	mainWindow: Page;
 };
 
@@ -43,6 +44,16 @@ export const test = base.extend<JoplinFixtures>({
 		await electronApp.close();
 	},
 
+	startupPluginsLoaded: async ({ electronApp }, use) => {
+		const startupPluginsLoadedPromise = electronApp.evaluate(({ ipcMain }) => {
+			return new Promise<void>(resolve => {
+				ipcMain.once('startup-plugins-loaded', () => resolve());
+			});
+		});
+
+		await use(startupPluginsLoadedPromise);
+	},
+
 	mainWindow: async ({ electronApp }, use) => {
 		const mainWindow = await firstNonDevToolsWindow(electronApp);
 		await use(mainWindow);

packages/app-desktop/package.json

@@ -7,12 +7,11 @@
   "scripts": {
     "dist": "yarn run electronRebuild && npx electron-builder",
     "build": "gulp build",
-    "postinstall": "yarn run build",
     "electronBuilder": "gulp electronBuilder",
     "electronRebuild": "gulp electronRebuild",
     "tsc": "tsc --project tsconfig.json",
     "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json",
-    "start": "gulp build && electron . --env dev --log-level debug --open-dev-tools",
+    "start": "gulp before-start && electron . --env dev --log-level debug --open-dev-tools",
     "test": "jest",
     "test-ui": "playwright test",
     "test-ci": "yarn test && sh ./integration-tests/run-ci.sh",
@@ -36,7 +35,8 @@
     "extraResources": [
       "build/icons/**",
       "build/images/**",
-      "build/defaultPlugins/**"
+      "build/defaultPlugins/**",
+      "build/7zip/**"
     ],
     "afterAllArtifactBuild": "./generateSha512.js",
     "asar": true,
@@ -115,7 +115,9 @@
   },
   "homepage": "https://github.com/laurent22/joplin#readme",
   "devDependencies": {
+    "7zip-bin": "5.2.0",
     "@electron/rebuild": "3.3.0",
+    "@joplin/default-plugins": "~2.13",
     "@joplin/tools": "~2.13",
     "@playwright/test": "1.39.0",
     "@testing-library/react-hooks": "8.0.1",
@@ -133,13 +135,9 @@
     "js-sha512": "0.8.0",
     "nan": "2.18.0",
     "react-test-renderer": "18.2.0",
+    "ts-node": "10.9.1",
     "typescript": "5.2.2"
   },
-  "optionalDependencies": {
-    "7zip-bin-linux": "^1.0.1",
-    "7zip-bin-mac": "^1.0.1",
-    "7zip-bin-win": "^2.1.1"
-  },
   "dependencies": {
     "@electron/notarize": "2.1.0",
     "@electron/remote": "2.0.12",

packages/app-desktop/services/plugins/plugin_index.js

@@ -2,6 +2,7 @@
 	// TODO: Not sure if that will work once packaged in Electron
 	const sandboxProxy = require('../../vendor/lib/@joplin/lib/services/plugins/sandboxProxy.js');
 	const ipcRenderer = require('electron').ipcRenderer;
+	const nodePath = require('path');
 
 	const ipcRendererSend = (message, args) => {
 		try {
@@ -56,7 +57,27 @@
 				return require('../../node_modules/@joplin/lib/node_modules/sqlite3/lib/sqlite3.js');
 			}
 
-			if (['fs-extra'].includes(modulePath)) return require(modulePath);
+			if (modulePath === 'fs-extra') {
+				return require('fs-extra');
+			}
+
+			// 7zip-bin is required by one of the default plugins (simple-backup)
+			if (modulePath === '7zip-bin') {
+				// 7zip-bin is very large -- return the path to a version of 7zip
+				// copied from 7zip-bin.
+				const executableName = process.platform === 'win32' ? '7za.exe' : '7za';
+
+				let rootDir = nodePath.dirname(nodePath.dirname(__dirname));
+
+				// When bundled, __dirname points to a file within app.asar. The build/ directory
+				// is outside of app.asar, and thus, we need an extra dirname(...).
+				if (nodePath.basename(rootDir).startsWith('app.asar')) {
+					rootDir = nodePath.dirname(rootDir);
+				}
+
+				const pathTo7za = nodePath.join(rootDir, 'build', '7zip', executableName);
+				return { path7za: nodePath.resolve(pathTo7za) };
+			}
 
 			throw new Error(`Module not found: ${modulePath}`);
 		}

packages/app-desktop/tools/copy7Zip.ts

@@ -0,0 +1,40 @@
+
+import { copy } from 'fs-extra';
+import { dirname, join } from 'path';
+
+const copy7Zip = async () => {
+	// We allow buildin for a different architecture/platform with
+	// the npm_config_target_arch and npm_config_target_platform environment variables.
+	//
+	// These are the same environment variables used by yarn when downloading dependencies.
+	//
+	const targetArch = process.env['npm_config_target_arch'] || process.arch;
+	const targetPlatform = process.env['npm_config_target_platform'] || process.platform;
+
+	console.info('Copying 7zip for platform', targetPlatform, 'and architecture', targetArch);
+
+	// To use the custom architecture/platform, we copy the relevant files from 7zip-bin
+	// directly:
+
+	const sevenZipBinDirectory = dirname(require.resolve('7zip-bin'));
+	const platformToSubdirectory: Record<string, string> = {
+		'win32': 'win',
+		'darwin': 'mac',
+		'linux': 'linux',
+	};
+
+	if (!(targetPlatform in platformToSubdirectory)) {
+		throw new Error(`Invalid target platform ${targetPlatform}. Must be in ${Object.keys(platformToSubdirectory)}`);
+	}
+
+	const fileName = targetPlatform === 'win32' ? '7za.exe' : '7za';
+	const pathTo7za = join(
+		sevenZipBinDirectory, platformToSubdirectory[targetPlatform], targetArch, fileName,
+	);
+
+	const rootDir = dirname(__dirname);
+	const outputPath = join(rootDir, 'build', '7zip', fileName);
+	await copy(pathTo7za, outputPath);
+};
+
+export default copy7Zip;

packages/app-desktop/tsconfig.json

@@ -7,5 +7,9 @@
 	"exclude": [
 		"**/node_modules",
 		"**/dist",
+
+		// Exclude gulpfile.ts to prevent Gulp from trying to build from
+		// gulpfile.js.
+		"gulpfile.ts"
 	],
 }

packages/default-plugins/.eslintignore

@@ -0,0 +1,2 @@
+plugin-base-repo/
+plugin-sources/*

packages/default-plugins/.gitignore

@@ -0,0 +1,2 @@
+built-plugins/
+plugin-sources/*

packages/default-plugins/build.ts

@@ -0,0 +1,31 @@
+import buildAll from './commands/buildAll';
+import editPatch from './commands/editPatch';
+const yargs = require('yargs');
+
+
+const build = () => {
+	yargs
+		.usage('$0 <cmd> [args]')
+		.command('build <outputDir>', 'build all', (yargs: any) => {
+			yargs.positional('outputDir', {
+				type: 'string',
+				describe: 'Path to the parent directory for built output',
+			});
+		}, async (args: any) => {
+			await buildAll(args.outputDir);
+			process.exit(0);
+		})
+		.command('patch <plugin>', 'Edit the patch file for the given plugin ID', (yargs: any) => {
+			yargs.positional('plugin', {
+				type: 'string',
+				describe: 'ID of the plugin to patch',
+			});
+		}, async (args: any) => {
+			await editPatch(args.plugin, null);
+			process.exit(0);
+		})
+		.help()
+		.argv;
+};
+
+build();

packages/default-plugins/buildDefaultPlugins.ts

@@ -0,0 +1,124 @@
+
+/* eslint-disable no-console */
+
+import { copy, exists, remove, mkdirp, readdir, mkdtemp, readFile, writeFile } from 'fs-extra';
+import { join, resolve, basename } from 'path';
+import { tmpdir } from 'os';
+import { chdir, cwd } from 'process';
+import { execCommand } from '@joplin/utils';
+import { glob } from 'glob';
+import readRepositoryJson from './utils/readRepositoryJson';
+import waitForCliInput from './utils/waitForCliInput';
+import getPathToPatchFileFor from './utils/getPathToPatchFileFor';
+
+type BeforeEachInstallCallback = (buildDir: string, pluginName: string)=> Promise<void>;
+
+const buildDefaultPlugins = async (outputParentDir: string|null, beforeInstall: BeforeEachInstallCallback) => {
+	const pluginSourcesDir = resolve(join(__dirname, 'plugin-sources'));
+	const pluginRepositoryData = await readRepositoryJson(join(__dirname, 'pluginRepositories.json'));
+
+	const originalDirectory = cwd();
+
+	const logStatus = (...message: string[]) => {
+		const blue = '\x1b[96m';
+		const reset = '\x1b[0m';
+		console.log(blue, ...message, reset);
+	};
+
+	for (const pluginId in pluginRepositoryData) {
+		const repositoryData = pluginRepositoryData[pluginId];
+
+		const buildDir = await mkdtemp(join(tmpdir(), 'default-plugin-build'));
+		try {
+			logStatus('Building plugin', pluginId, 'at', buildDir);
+			const pluginDir = resolve(join(pluginSourcesDir, pluginId));
+
+			// Clone the repository if not done yet
+			if (!(await exists(pluginDir)) || (await readdir(pluginDir)).length === 0) {
+				logStatus(`Cloning from repository ${repositoryData.cloneUrl}`);
+				await execCommand(['git', 'clone', '--', repositoryData.cloneUrl, pluginDir]);
+				chdir(pluginDir);
+			}
+
+			chdir(pluginDir);
+			const currentCommitHash = (await execCommand(['git', 'rev-parse', 'HEAD~'])).trim();
+			const expectedCommitHash = repositoryData.commit;
+
+			if (currentCommitHash !== expectedCommitHash) {
+				logStatus(`Switching to commit ${expectedCommitHash}`);
+				await execCommand(['git', 'switch', repositoryData.branch]);
+				await execCommand(['git', 'checkout', expectedCommitHash]);
+			}
+
+			logStatus('Copying repository files...');
+			await copy(pluginDir, buildDir, {
+				filter: fileName => {
+					return basename(fileName) !== '.git';
+				},
+			});
+
+			chdir(buildDir);
+
+			logStatus('Initializing repository.');
+			await execCommand('git init . -b main');
+
+			logStatus('Marking manifest as built-in');
+			const manifestFile = './src/manifest.json';
+			const manifest = JSON.parse(await readFile(manifestFile, 'utf8'));
+			manifest._built_in = true;
+			await writeFile(manifestFile, JSON.stringify(manifest, undefined, '\t'));
+
+			logStatus('Creating initial commit.');
+			await execCommand('git add .');
+			await execCommand(['git', 'config', 'user.name', 'Build script']);
+			await execCommand(['git', 'config', 'user.email', '']);
+			await execCommand(['git', 'commit', '-m', 'Initial commit']);
+
+			const patchFile = getPathToPatchFileFor(pluginId);
+			if (await exists(patchFile)) {
+				logStatus('Applying patch.');
+				await execCommand(['git', 'apply', patchFile]);
+			}
+
+			await beforeInstall(buildDir, pluginId);
+
+			logStatus('Installing dependencies.');
+			await execCommand('npm install');
+
+			const jplFiles = await glob('publish/*.jpl');
+			logStatus(`Found built .jpl files: ${JSON.stringify(jplFiles)}`);
+
+			if (jplFiles.length === 0) {
+				throw new Error(`No published files found in ${buildDir}/publish`);
+			}
+
+			if (outputParentDir !== null) {
+				logStatus(`Checking output directory in ${outputParentDir}`);
+				const outputDirectory = join(outputParentDir, pluginId);
+				if (await exists(outputDirectory)) {
+					await remove(outputDirectory);
+				}
+				await mkdirp(outputDirectory);
+
+				const sourceFile = jplFiles[0];
+				const destFile = join(outputDirectory, 'plugin.jpl');
+
+				logStatus(`Copying built file from ${sourceFile} to ${destFile}`);
+				await copy(sourceFile, destFile);
+			} else {
+				console.warn('No output directory specified. Not copying built .jpl files.');
+			}
+		} catch (error) {
+			console.error(error);
+			console.log('Build directory', buildDir);
+			await waitForCliInput();
+			throw error;
+		} finally {
+			chdir(originalDirectory);
+			await remove(buildDir);
+			logStatus('Removed build directory');
+		}
+	}
+};
+
+export default buildDefaultPlugins;

packages/default-plugins/commands/buildAll.ts

@@ -0,0 +1,7 @@
+import buildDefaultPlugins from '../buildDefaultPlugins';
+
+const buildAll = (outputDirectory: string) => {
+	return buildDefaultPlugins(outputDirectory, async () => { });
+};
+
+export default buildAll;

packages/default-plugins/commands/editPatch.ts

@@ -0,0 +1,31 @@
+import { execCommand } from '@joplin/utils';
+import waitForCliInput from '../utils/waitForCliInput';
+import { copy } from 'fs-extra';
+import { join } from 'path';
+import buildDefaultPlugins from '../buildDefaultPlugins';
+import getPathToPatchFileFor from '../utils/getPathToPatchFileFor';
+
+const editPatch = async (targetPluginId: string, outputParentDir: string|null) => {
+	let patchedPlugin = false;
+
+	await buildDefaultPlugins(outputParentDir, async (buildDir, pluginId) => {
+		if (pluginId !== targetPluginId) {
+			return;
+		}
+
+		// eslint-disable-next-line no-console
+		console.log('Make changes to', buildDir, 'to create a patch.');
+		await waitForCliInput();
+		await execCommand(['sh', '-c', 'git diff -p > diff.diff']);
+
+		await copy(join(buildDir, './diff.diff'), getPathToPatchFileFor(pluginId));
+
+		patchedPlugin = true;
+	});
+
+	if (!patchedPlugin) {
+		throw new Error(`No default plugin with ID ${targetPluginId} found!`);
+	}
+};
+
+export default editPatch;

packages/default-plugins/package.json

@@ -0,0 +1,25 @@
+{
+	"name": "@joplin/default-plugins",
+	"version": "2.13.0",
+	"description": "Default plugins bundler",
+	"private": true,
+	"scripts": {
+		"tsc": "tsc --project tsconfig.json",
+		"watch": "tsc --watch --preserveWatchOutput --project tsconfig.json",
+		"patch": "ts-node build.ts patch"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/laurent22/joplin.git"
+	},
+	"devDependencies": {
+		"@types/yargs": "17.0.31",
+		"ts-node": "10.9.1",
+		"typescript": "5.2.2"
+	},
+	"dependencies": {
+		"@joplin/utils": "~2.13",
+		"fs-extra": "11.1.1",
+		"yargs": "17.7.2"
+	}
+}

packages/default-plugins/plugin-patches/io.github.jackgruber.backup.diff

@@ -0,0 +1,62 @@
+diff --git a/src/sevenZip.ts b/src/sevenZip.ts
+index ef2a527..d98c777 100644
+--- a/src/sevenZip.ts
++++ b/src/sevenZip.ts
+@@ -1,21 +1,21 @@
+ // https://sevenzip.osdn.jp/chm/cmdline/exit_codes.htm
+ // https://sevenzip.osdn.jp/chm/cmdline/commands/index.htm
+ import * as _7z from "node-7z";
+-import * as sevenBin from "7zip-bin";
+-import * as path from "path";
+ import { exec } from "child_process";
+ import joplin from "api";
+-
+-export let pathTo7zip = sevenBin.path7za;
+-
+-export namespace sevenZip {
+-  export async function updateBinPath() {
+-    pathTo7zip = path.join(
+-      await joplin.plugins.installationDir(),
+-      "7zip-bin",
+-      pathTo7zip
+-    );
+-  }
++const sevenBin = joplin.require("7zip-bin");
++
++ export let pathTo7zip = sevenBin.path7za;
++
++ export namespace sevenZip {
++   export async function updateBinPath() {
++    // Not necessary with 7zip required from Joplin
++    // pathTo7zip = path.join(
++    //   await joplin.plugins.installationDir(),
++    //   "7zip-bin",
++    //   pathTo7zip
++    // );
++   }
+ 
+   export async function setExecutionFlag() {
+     if (process.platform !== "win32") {
+diff --git a/webpack.config.js b/webpack.config.js
+index 34a1797..7b2a480 100644
+--- a/webpack.config.js
++++ b/webpack.config.js
+@@ -200,15 +200,9 @@ const pluginConfig = { ...baseConfig, entry: './src/index.ts',
+ 		path: distDir,
+ 	},
+ 	plugins: [
+-		new CopyPlugin({
+-			patterns: [
+-				{
+-					from: '**/*',
+-					context: path.resolve(__dirname, 'node_modules','7zip-bin'),
+-					to: path.resolve(__dirname, 'dist/7zip-bin/'),
+-				},
+-			]
+-		}),
++		// Removed a CopyPlugin (added by Simple Backup, not necessary when using
++		// Joplin's built-in 7zip)
++
+ 		new CopyPlugin({
+ 			patterns: [
+ 				{

packages/default-plugins/pluginRepositories.json

@@ -0,0 +1,7 @@
+{
+	"io.github.jackgruber.backup": {
+		"cloneUrl": "https://github.com/JackGruber/joplin-plugin-backup.git",
+		"branch": "master",
+		"commit": "021085cc37ed83a91a7950744e462782e27c04a6"
+	}
+}

packages/default-plugins/tsconfig.json

@@ -0,0 +1,12 @@
+{
+	"extends": "../../tsconfig.json",
+	"include": [
+		"**/*.ts",
+		"**/*.tsx",
+	],
+	"exclude": [
+		"**/node_modules",
+		"plugin-sources/",
+		"plugin-base-repo/"
+	]
+}

packages/default-plugins/utils/getPathToPatchFileFor.ts

@@ -0,0 +1,9 @@
+
+import { join, dirname } from 'path';
+
+const getPathToPatchFileFor = (pluginName: string) => {
+	const rootDir = dirname(__dirname);
+	return join(rootDir, 'plugin-patches', `${pluginName}.diff`);
+};
+
+export default getPathToPatchFileFor;

packages/default-plugins/utils/readRepositoryJson.ts

@@ -0,0 +1,37 @@
+import { readFile } from 'fs-extra';
+
+export interface RepositoryData {
+	cloneUrl: string;
+	branch: string;
+	commit: string;
+}
+
+export interface AllRepositoryData {
+	[pluginId: string]: RepositoryData;
+}
+
+const readRepositoryJson = async (repositoryDataFilepath: string): Promise<AllRepositoryData> => {
+	const fileContent = await readFile(repositoryDataFilepath, 'utf8');
+	const parsedJson = JSON.parse(fileContent);
+
+	// Validate
+	for (const pluginId in parsedJson) {
+		if (typeof parsedJson[pluginId] !== 'object') {
+			throw new Error('pluginRepositories should map from plugin IDs to objects.');
+		}
+
+		const assertPropertyIsString = (propertyName: string) => {
+			if (typeof parsedJson[pluginId][propertyName] !== 'string') {
+				throw new Error(`Plugin ${pluginId} should have field '${propertyName}' of type string.`);
+			}
+		};
+
+		assertPropertyIsString('cloneUrl');
+		assertPropertyIsString('branch');
+		assertPropertyIsString('commit');
+	}
+
+	return parsedJson;
+};
+
+export default readRepositoryJson;

packages/default-plugins/utils/waitForCliInput.ts

@@ -0,0 +1,23 @@
+
+const readline = require('readline/promises');
+
+/* eslint-disable no-console */
+
+let readlineInterface: any = null;
+const waitForCliInput = async () => {
+	readlineInterface ??= readline.createInterface({
+		input: process.stdin,
+		output: process.stdout,
+	});
+	if (process.stdin.isTTY) {
+		const green = '\x1b[92m';
+		const reset = '\x1b[0m';
+		await readlineInterface.question(`${green}[Press enter to continue]${reset}`);
+
+		console.log('Continuing...');
+	} else {
+		console.warn('Input is not from a TTY -- not waiting for input.');
+	}
+};
+
+export default waitForCliInput;

packages/lib/services/plugins/PluginService.ts

@@ -33,7 +33,6 @@ export interface SettingAndValue {
 }
 
 export interface DefaultPluginSettings {
-	version: string;
 	settings?: SettingAndValue;
 }
 

packages/lib/services/plugins/defaultPlugins/defaultPluginsUtils.ts

@@ -35,7 +35,10 @@ export async function installDefaultPlugins(service: PluginService, defaultPlugi
 		const pluginId = pluginStat.path;
 
 		// if pluginId is present in 'installedDefaultPlugins' array or it doesn't have default plugin ID, then we won't install it again as default plugin
-		if (installedPlugins.includes(pluginId) || !defaultPluginsId.includes(pluginId)) continue;
+		if (installedPlugins.includes(pluginId) || !defaultPluginsId.includes(pluginId)) {
+			logger.debug(`Skipping default plugin ${pluginId}, ${!defaultPluginsId.includes(pluginId) ? '(Not a default)' : ''}`);
+			continue;
+		}
 		const defaultPluginPath: string = path.join(defaultPluginsDir, pluginId, 'plugin.jpl');
 		await service.installPlugin(defaultPluginPath, false);
 

packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.ts

@@ -4,14 +4,10 @@ import Setting from '../../../models/Setting';
 const getDefaultPluginsInfo = (): DefaultPluginsInfo => {
 	const defaultPlugins = {
 		'io.github.jackgruber.backup': {
-			version: '1.1.1',
 			settings: {
 				'path': `${Setting.value('profileDir')}`,
 			},
 		},
-		'plugin.calebjohn.rich-markdown': {
-			version: '0.8.3',
-		},
 	};
 	return defaultPlugins;
 };

packages/tools/bundleDefaultPlugins.test.ts

@@ -1,206 +0,0 @@
-
-
-
-import { join } from 'path';
-import { downloadPlugins, extractPlugins, localPluginsVersion } from './bundleDefaultPlugins';
-import { pathExists, readFile, remove } from 'fs-extra';
-import Setting from '@joplin/lib/models/Setting';
-import { createTempDir, supportDir } from '@joplin/lib/testing/test-utils';
-import { rootDir } from './tool-utils';
-
-const fetch = require('node-fetch');
-
-jest.mock('node-fetch', ()=>jest.fn());
-
-const manifests = {
-	'io.github.jackgruber.backup': {
-		'manifest_version': 1,
-		'id': 'io.github.jackgruber.backup',
-		'app_min_version': '2.1.3',
-		'version': '1.1.0',
-		'name': 'Simple Backup',
-		'description': 'Plugin to create manual and automatic backups.',
-		'author': 'JackGruber',
-		'homepage_url': 'https://github.com/JackGruber/joplin-plugin-backup/blob/master/README.md',
-		'repository_url': 'https://github.com/JackGruber/joplin-plugin-backup',
-		'keywords': [
-			'backup',
-			'jex',
-			'export',
-			'zip',
-			'7zip',
-			'encrypted',
-		],
-		'_publish_hash': 'sha256:8d8c6a3bb92fafc587269aea58b623b05242d42c0766a05bbe25c3ba2bbdf8ee',
-		'_publish_commit': 'master:00ed52133c659e0f3ac1a55f70b776c42fca0a6d',
-		'_npm_package_name': 'joplin-plugin-backup',
-	},
-	'plugin.calebjohn.rich-markdown': {
-		'manifest_version': 1,
-		'id': 'plugin.calebjohn.rich-markdown',
-		'app_min_version': '2.7',
-		'version': '0.9.0',
-		'name': 'Rich Markdown',
-		'description': 'Helping you ditch the markdown viewer for good.',
-		'author': 'Caleb John',
-		'homepage_url': 'https://github.com/CalebJohn/joplin-rich-markdown#readme',
-		'repository_url': 'https://github.com/CalebJohn/joplin-rich-markdown',
-		'keywords': [
-			'editor',
-			'visual',
-		],
-		'_publish_hash': 'sha256:95337a3868aebdc9bf8c347a37460d0c2753b391ff51a0c72bdccdef9679705f',
-		'_publish_commit': 'main:af3493b6ca96c931327ab3bd04906faaed0c782c',
-		'_npm_package_name': 'joplin-plugin-rich-markdown',
-	},
-
-};
-
-const NPM_Response1 = JSON.stringify({
-	'_id': 'joplin-plugin-rich-markdown',
-	'name': 'joplin-plugin-rich-markdown',
-	'versions': {
-		'0.8.2': {
-			'name': 'joplin-plugin-rich-markdown',
-			'version': '0.8.2',
-			'description': 'A plugin that will finally allow you to ditch the markdown viewer, saving space and making your life easier.',
-			'_id': 'joplin-plugin-rich-markdown@0.1.0',
-			'dist': {
-				'tarball': 'no-link-here',
-			},
-		},
-		'0.9.0': {
-			'name': 'joplin-plugin-rich-markdown',
-			'version': '0.9.0',
-			'dist': {
-				'tarball': 'response-1-link',
-			},
-		},
-	},
-});
-
-const NPM_Response2 = JSON.stringify({
-	'_id': 'io.github.jackgruber.backup',
-	'name': 'joplin-plugin-rich-markdown',
-	'versions': {
-		'1.0.0': {
-			'name': 'joplin-plugin-rich-markdown',
-			'version': '1.0.0',
-			'description': 'A plugin that will finally allow you to ditch the markdown viewer, saving space and making your life easier.',
-			'_id': 'joplin-plugin-rich-markdown@0.1.0',
-			'dist': {
-				'tarball': 'no-link-here',
-			},
-		},
-		'1.1.0': {
-			'name': 'joplin-plugin-rich-markdown',
-			'version': '1.1.0',
-			'dist': {
-				'tarball': 'response-2-link',
-			},
-		},
-	},
-});
-
-async function mockPluginData() {
-	const filePath = join(__dirname, '..', 'app-cli', 'tests', 'services', 'plugins', 'mockData', 'mockPlugin.tgz');
-	const tgzData = await readFile(filePath, 'utf8');
-	return tgzData;
-}
-
-describe('bundleDefaultPlugins', () => {
-
-	const testDefaultPluginsInfo = {
-		'plugin.calebjohn.rich-markdown': {
-			version: '0.9.0',
-		},
-		'io.github.jackgruber.backup': {
-			version: '1.1.0',
-			settings: {
-				'path': `${Setting.value('profileDir')}`,
-			},
-		},
-	};
-
-	it('should get local plugin versions', async () => {
-		const manifestsPath = join(supportDir, 'pluginRepo', 'plugins');
-		const testDefaultPluginsInfo = {
-			'joplin.plugin.ambrt.backlinksToNote': { version: '1.0.4' },
-			'org.joplinapp.plugins.ToggleSidebars': { version: '1.0.2' },
-		};
-		const localPluginsVersions = await localPluginsVersion(manifestsPath, testDefaultPluginsInfo);
-
-		expect(localPluginsVersions['joplin.plugin.ambrt.backlinksToNote']).toBe('1.0.4');
-		expect(localPluginsVersions['org.joplinapp.plugins.ToggleSidebars']).toBe('1.0.2');
-	});
-
-	it('should download plugins folder from GitHub with no initial plugins', async () => {
-
-		const testCases = [
-			{
-				localVersions: { 'io.github.jackgruber.backup': '0.0.0', 'plugin.calebjohn.rich-markdown': '0.0.0' },
-				downloadedPlugin1: 'joplin-plugin-rich-markdown-0.9.0.tgz',
-				downloadedPlugin2: 'joplin-plugin-backup-1.1.0.tgz',
-				numberOfCalls: 4,
-				calledWith: ['https://registry.npmjs.org/joplin-plugin-rich-markdown', 'response-1-link', 'https://registry.npmjs.org/joplin-plugin-backup', 'response-2-link'],
-			},
-			{
-				localVersions: { 'io.github.jackgruber.backup': '1.1.0', 'plugin.calebjohn.rich-markdown': '0.0.0' },
-				downloadedPlugin1: 'joplin-plugin-rich-markdown-0.9.0.tgz',
-				downloadedPlugin2: undefined,
-				numberOfCalls: 2,
-				calledWith: ['https://registry.npmjs.org/joplin-plugin-rich-markdown', 'response-1-link'],
-			},
-			{
-				localVersions: { 'io.github.jackgruber.backup': '1.1.0', 'plugin.calebjohn.rich-markdown': '0.9.0' },
-				downloadedPlugin1: undefined,
-				downloadedPlugin2: undefined,
-				numberOfCalls: 0,
-				calledWith: [],
-			},
-		];
-
-		const tgzData = await mockPluginData();
-
-		const mockFetch = fetch as jest.MockedFunction<typeof fetch>;
-
-		for (const testCase of testCases) {
-
-			mockFetch.mockResolvedValueOnce({ text: () => Promise.resolve(NPM_Response1), ok: true })
-				.mockResolvedValueOnce({ buffer: () => Promise.resolve(tgzData), ok: true })
-				.mockResolvedValueOnce({ text: () => Promise.resolve(NPM_Response2), ok: true })
-				.mockResolvedValueOnce({ buffer: () => Promise.resolve(tgzData), ok: true });
-
-			const downloadedPlugins = await downloadPlugins(testCase.localVersions, testDefaultPluginsInfo, manifests);
-
-			expect(downloadedPlugins[Object.keys(testDefaultPluginsInfo)[0]]).toBe(testCase.downloadedPlugin1);
-			expect(downloadedPlugins[Object.keys(testDefaultPluginsInfo)[1]]).toBe(testCase.downloadedPlugin2);
-
-			expect(mockFetch).toHaveBeenCalledTimes(testCase.numberOfCalls);
-
-			// eslint-disable-next-line github/array-foreach -- Old code before rule was applied
-			testCase.calledWith.forEach((callValue, index) => expect(mockFetch).toHaveBeenNthCalledWith(index + 1, callValue));
-
-			jest.clearAllMocks();
-		}
-
-		await remove(`${rootDir}/packages/tools/joplin-plugin-backup-1.1.0.tgz`);
-		await remove(`${rootDir}/packages/tools/joplin-plugin-rich-markdown-0.9.0.tgz`);
-	});
-
-	it('should extract plugins files', async () => {
-
-		const downloadedPluginsNames = { 'plugin.calebjohn.rich-markdown': 'mockPlugin.tgz' };
-
-		const filePath = join(__dirname, '..', 'app-cli', 'tests', 'services', 'plugins', 'mockData');
-		const tempDir = await createTempDir();
-
-		await extractPlugins(filePath, tempDir, downloadedPluginsNames);
-
-		expect(await pathExists(join(tempDir, 'plugin.calebjohn.rich-markdown', 'plugin.jpl'))).toBe(true);
-		expect(await pathExists(join(tempDir, 'plugin.calebjohn.rich-markdown', 'manifest.json'))).toBe(true);
-
-		await remove(tempDir);
-	});
-
-});

packages/tools/bundleDefaultPlugins.ts

@@ -1,96 +0,0 @@
-import { join } from 'path';
-import { pathExists, mkdir, readFile, move, remove, writeFile } from 'fs-extra';
-import { DefaultPluginsInfo } from '@joplin/lib/services/plugins/PluginService';
-import getDefaultPluginsInfo from '@joplin/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo';
-import { execCommand } from '@joplin/utils';
-const fetch = require('node-fetch');
-
-interface PluginAndVersion {
-	[pluginId: string]: string;
-}
-
-interface PluginIdAndName {
-	[pluginId: string]: string;
-}
-
-export const localPluginsVersion = async (defaultPluginDir: string, defaultPluginsInfo: DefaultPluginsInfo): Promise<PluginAndVersion> => {
-	if (!await pathExists(join(defaultPluginDir))) await mkdir(defaultPluginDir);
-	const localPluginsVersions: PluginAndVersion = {};
-
-	for (const pluginId of Object.keys(defaultPluginsInfo)) {
-
-		if (!await pathExists(join(defaultPluginDir, pluginId))) {
-			localPluginsVersions[pluginId] = '0.0.0';
-			continue;
-		}
-		const data = await readFile(`${defaultPluginDir}/${pluginId}/manifest.json`, 'utf8');
-		const manifest = JSON.parse(data);
-		localPluginsVersions[pluginId] = manifest.version;
-	}
-	return localPluginsVersions;
-};
-
-async function downloadFile(url: string, outputPath: string) {
-	const response = await fetch(url);
-	if (!response.ok) {
-		const responseText = await response.text();
-		throw new Error(`Cannot download file from ${url} : ${responseText.substr(0, 500)}`);
-	}
-	await writeFile(outputPath, await response.buffer());
-}
-
-export async function extractPlugins(currentDir: string, defaultPluginDir: string, downloadedPluginsNames: PluginIdAndName): Promise<void> {
-	for (const pluginId of Object.keys(downloadedPluginsNames)) {
-		await execCommand(`tar xzf ${currentDir}/${downloadedPluginsNames[pluginId]}`, { quiet: true });
-		await move(`package/publish/${pluginId}.jpl`, `${defaultPluginDir}/${pluginId}/plugin.jpl`, { overwrite: true });
-		await move(`package/publish/${pluginId}.json`, `${defaultPluginDir}/${pluginId}/manifest.json`, { overwrite: true });
-		await remove(`${downloadedPluginsNames[pluginId]}`);
-		await remove('package');
-	}
-}
-
-export const downloadPlugins = async (localPluginsVersions: PluginAndVersion, defaultPluginsInfo: DefaultPluginsInfo, manifests: any): Promise<PluginIdAndName> => {
-
-	const downloadedPluginsNames: PluginIdAndName = {};
-	for (const pluginId of Object.keys(defaultPluginsInfo)) {
-		if (localPluginsVersions[pluginId] === defaultPluginsInfo[pluginId].version) continue;
-		const response = await fetch(`https://registry.npmjs.org/${manifests[pluginId]._npm_package_name}`);
-
-		if (!response.ok) {
-			const responseText = await response.text();
-			throw new Error(`Cannot fetch ${manifests[pluginId]._npm_package_name} release info from NPM : ${responseText.substr(0, 500)}`);
-		}
-		const releaseText = await response.text();
-		const release = JSON.parse(releaseText);
-
-		const pluginUrl = release.versions[defaultPluginsInfo[pluginId].version].dist.tarball;
-
-		const pluginName = `${manifests[pluginId]._npm_package_name}-${defaultPluginsInfo[pluginId].version}.tgz`;
-		await downloadFile(pluginUrl, pluginName);
-
-		downloadedPluginsNames[pluginId] = pluginName;
-	}
-	return downloadedPluginsNames;
-};
-
-async function start(): Promise<void> {
-	const defaultPluginDir = join(__dirname, '..', '..', 'packages', 'app-desktop', 'build', 'defaultPlugins');
-	const defaultPluginsInfo = getDefaultPluginsInfo();
-
-	const manifestData = await fetch('https://raw.githubusercontent.com/joplin/plugins/master/manifests.json');
-	const manifests = JSON.parse(await manifestData.text());
-	if (!manifests) throw new Error('Invalid or missing JSON');
-
-	const localPluginsVersions = await localPluginsVersion(defaultPluginDir, defaultPluginsInfo);
-	const downloadedPluginNames: PluginIdAndName = await downloadPlugins(localPluginsVersions, defaultPluginsInfo, manifests);
-	await extractPlugins(__dirname, defaultPluginDir, downloadedPluginNames);
-}
-
-if (require.main === module) {
-// eslint-disable-next-line promise/prefer-await-to-then -- Old code before rule was applied
-	start().catch((error) => {
-		console.error('Fatal error');
-		console.error(error);
-		process.exit(1);
-	});
-}

packages/tools/gulp/tasks/updateIgnoredTypeScriptBuild.js

@@ -31,6 +31,7 @@ module.exports = {
 				'packages/app-desktop/dist/**',
 				'packages/app-mobile/android/**',
 				'packages/app-mobile/ios/**',
+				'packages/default-plugins/plugin-sources/**',
 				'packages/fork-sax/**',
 				'packages/lib/plugin_types/**',
 				'packages/server/**',

packages/tools/setupNewRelease.ts

@@ -140,6 +140,7 @@ async function main() {
 	await updatePackageVersion(`${rootDir}/packages/server/package.json`, majorMinorVersion, options);
 	await updatePackageVersion(`${rootDir}/packages/tools/package.json`, majorMinorVersion, options);
 	await updatePackageVersion(`${rootDir}/packages/utils/package.json`, majorMinorVersion, options);
+	await updatePackageVersion(`${rootDir}/packages/default-plugins/package.json`, majorMinorVersion, options);
 	await updatePackageVersion(`${rootDir}/packages/editor/package.json`, majorMinorVersion, options);
 
 	if (options.updateVersion) {

packages/utils/package.json

@@ -19,6 +19,7 @@
   },
   "scripts": {
     "tsc": "tsc --project tsconfig.json",
+    "build": "yarn run tsc",
     "watch": "tsc --watch --preserveWatchOutput --project tsconfig.json",
     "test": "jest --verbose=false",
     "test-ci": "yarn test"

readme/dev/spec/default_plugins.md

@@ -9,7 +9,6 @@ To add a new default plugin for desktop:
 ```
 const defaultPlugins = {
     'samplePluginId': {
-        version: '1.0.0',
         settings: {
             'settingName1': 'setting-value1',
             'settingName2': 'setting-value2',
@@ -18,13 +17,37 @@ const defaultPlugins = {
 };
 ```
 
+After this, add the commit, branch, and clone URL to be build from to `pluginRepositories.json`.
+
+For example,
+```json
+{
+	"plugin.id.here": {
+		"cloneUrl": "https://example.com/plugin-repo/plugin-repo-here.git",
+		"branch": "main",
+		"commit": "840d2e84b70adf6de961e167dcd27ddad088b286"
+	}
+}
+```
+
+## Patching the plugin
+
+Some plugins need patching. To create or update a plugin's patch, run the `patch` command in the `packages/default-plugins/` directory.
+
+For example,
+```shell
+$ cd packages/default-plugins
+$ yarn run patch plugin.id.here
+```
+
+The script will create a temporary directory in which changes can be made. Do not stage the changes that should appear in the patch.
+
 ## Bundling of default plugins
 
-Script for bundling default plugins is present in [bundleDefaultPlugins.ts](https://github.com/laurent22/joplin/blob/eb7083d7888433ff6ef76ccfb7fb87ba951d513f/packages/tools/bundleDefaultPlugins.ts)
+Scripts for bundling default plugins are present in `packages/default-plugins/`.
 
-Every time a new desktop release is being built, we compare the local default plugins version with pinned plugin version mentioned in [desktopDefaultPluginsInfo.ts](https://github.com/laurent22/joplin/blob/eb7083d7888433ff6ef76ccfb7fb87ba951d513f/packages/lib/services/plugins/defaultPlugins/desktopDefaultPluginsInfo.ts)
+These are run by the `app-desktop` package on a full `build` (e.g. on `postinstall`).
 
-If there is a newer version available, we will pull the `tgz` file of plugin from NPM registry and extract it. We will then move `manifest.json` and `plugin.jpl` to the build folder of desktop.
 
 ## Installing of default plugins
 

yarn.lock

@@ -5,24 +5,10 @@ __metadata:
   version: 6
   cacheKey: 8
 
-"7zip-bin-linux@npm:^1.0.1":
+"7zip-bin@npm:5.2.0":
-  version: 1.3.1
+  version: 5.2.0
-  resolution: "7zip-bin-linux@npm:1.3.1"
+  resolution: "7zip-bin@npm:5.2.0"
-  conditions: os=linux
+  checksum: 85d3102275342f1f4ba7d17e778e526dee3dbec0f57d29be7afaa6e3c26687d40a6eccf520e9140143f85a51f3353f6b545f760eff3f776c6ffb30dc5252fb7c
-  languageName: node
-  linkType: hard
-
-"7zip-bin-mac@npm:^1.0.1":
-  version: 1.0.1
-  resolution: "7zip-bin-mac@npm:1.0.1"
-  conditions: os=darwin
-  languageName: node
-  linkType: hard
-
-"7zip-bin-win@npm:^2.1.1":
-  version: 2.2.0
-  resolution: "7zip-bin-win@npm:2.2.0"
-  conditions: os=win32
   languageName: node
   linkType: hard
 
@@ -6411,14 +6397,13 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@joplin/app-desktop@workspace:packages/app-desktop"
   dependencies:
-    7zip-bin-linux: ^1.0.1
+    7zip-bin: 5.2.0
-    7zip-bin-mac: ^1.0.1
-    7zip-bin-win: ^2.1.1
     "@electron/notarize": 2.1.0
     "@electron/rebuild": 3.3.0
     "@electron/remote": 2.0.12
     "@fortawesome/fontawesome-free": 5.15.4
     "@joeattardi/emoji-button": 4.6.4
+    "@joplin/default-plugins": ~2.13
     "@joplin/editor": ~2.13
     "@joplin/lib": ~2.13
     "@joplin/renderer": ~2.13
@@ -6478,14 +6463,8 @@ __metadata:
     styled-system: 5.1.5
     taboverride: 4.0.3
     tinymce: 5.10.6
+    ts-node: 10.9.1
     typescript: 5.2.2
-  dependenciesMeta:
-    7zip-bin-linux:
-      optional: true
-    7zip-bin-mac:
-      optional: true
-    7zip-bin-win:
-      optional: true
   languageName: unknown
   linkType: soft
 
@@ -6594,6 +6573,19 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@joplin/default-plugins@workspace:packages/default-plugins, @joplin/default-plugins@~2.13":
+  version: 0.0.0-use.local
+  resolution: "@joplin/default-plugins@workspace:packages/default-plugins"
+  dependencies:
+    "@joplin/utils": ~2.13
+    "@types/yargs": 17.0.31
+    fs-extra: 11.1.1
+    ts-node: 10.9.1
+    typescript: 5.2.2
+    yargs: 17.7.2
+  languageName: unknown
+  linkType: soft
+
 "@joplin/doc-builder@workspace:packages/doc-builder":
   version: 0.0.0-use.local
   resolution: "@joplin/doc-builder@workspace:packages/doc-builder"