Tools: Handle obsolete packages on plugin repo

package.json

@@ -13,6 +13,7 @@
     "buildApiDoc": "npm start --prefix=packages/app-cli -- apidoc ../../readme/api/references/rest_api.md",
     "buildDoc": "./packages/tools/build-all.sh",
     "buildPluginDoc": "typedoc --name 'Joplin Plugin API Documentation' --mode file -theme './Assets/PluginDocTheme/' --readme './Assets/PluginDocTheme/index.md' --excludeNotExported --excludeExternals --excludePrivate --excludeProtected --out docs/api/references/plugin_api packages/lib/services/plugins/api/",
+    "buildPluginRepo": "node packages/tools/build-plugin-repository.js",
     "buildTranslations": "npm run tsc && node packages/tools/build-translation.js",
     "buildTranslationsNoTsc": "node packages/tools/build-translation.js",
     "buildWebsite": "npm run buildApiDoc && node ./packages/tools/build-website.js && npm run buildPluginDoc",

packages/lib/markdownUtils.ts

@@ -30,6 +30,17 @@ const markdownUtils = {
 		return url;
 	},
 
+	escapeTableCell(text: string) {
+		// Disable HTML code
+		text = text.replace(/</g, '&lt;');
+		text = text.replace(/>/g, '&gt;');
+		// Table cells can't contain new lines so replace with <br/>
+		text = text.replace(/\n/g, '<br/>');
+		// "|" is a reserved characters that should be escaped
+		text = text.replace(/\|/g, '\\|');
+		return text;
+	},
+
 	unescapeLinkUrl(url: string) {
 		url = url.replace(/%28/g, '(');
 		url = url.replace(/%29/g, ')');
@@ -117,8 +128,8 @@ const markdownUtils = {
 			const rowMd = [];
 			for (let j = 0; j < headers.length; j++) {
 				const h = headers[j];
-				const value = h.filter ? h.filter(row[h.name]) : row[h.name];
+				const valueMd = markdownUtils.escapeTableCell(h.filter ? h.filter(row[h.name]) : row[h.name]);
-				rowMd.push(stringPadding(value, 3, ' ', stringPadding.RIGHT));
+				rowMd.push(stringPadding(valueMd, 3, ' ', stringPadding.RIGHT));
 			}
 			output.push(rowMd.join(' | '));
 		}

packages/tools/build-plugin-repository.ts

@@ -11,12 +11,23 @@ interface NpmPackage {
 	date: Date;
 }
 
+function stripOffPackageOrg(name: string): string {
+	const n = name.split('/');
+	if (n[0][0] === '@') n.splice(0, 1);
+	return n.join('/');
+}
+
+function isJoplinPluginPackage(pack: any): boolean {
+	if (!pack.keywords || !pack.keywords.includes('joplin-plugin')) return false;
+	if (stripOffPackageOrg(pack.name).indexOf('joplin-plugin') !== 0) return false;
+	return true;
+}
+
 function pluginInfoFromSearchResults(results: any[]): NpmPackage[] {
 	const output: NpmPackage[] = [];
 
 	for (const r of results) {
-		if (r.name.indexOf('joplin-plugin') !== 0) continue;
+		if (!isJoplinPluginPackage(r)) continue;
-		if (!r.keywords || !r.keywords.includes('joplin-plugin')) continue;
 
 		output.push({
 			name: r.name,
@@ -48,7 +59,14 @@ async function readJsonFile(manifestPath: string, defaultValue: any = null): Pro
 	return JSON.parse(content);
 }
 
-async function extractPluginFilesFromPackage(originalPluginManifests: any, workDir: string, packageName: string, destDir: string): Promise<any> {
+function caseInsensitiveFindManifest(manifests: any, manifestId: string): any {
+	for (const id of Object.keys(manifests)) {
+		if (id.toLowerCase() === manifestId.toLowerCase()) return manifests[id];
+	}
+	return null;
+}
+
+async function extractPluginFilesFromPackage(existingManifests: any, workDir: string, packageName: string, destDir: string): Promise<any> {
 	const previousDir = process.cwd();
 	process.chdir(workDir);
 
@@ -76,7 +94,11 @@ async function extractPluginFilesFromPackage(originalPluginManifests: any, workD
 	// package name, we skip it. Otherwise it would allow anyone to overwrite
 	// someone else plugin just by using the same ID. So the first plugin with
 	// this ID that was originally added is kept.
-	const originalManifest = originalPluginManifests[manifest.id];
+	//
+	// We need case insensitive match because the filesystem might be case
+	// insensitive too.
+	const originalManifest = caseInsensitiveFindManifest(existingManifests, manifest.id);
+
 	if (originalManifest && originalManifest._npm_package_name !== packageName) {
 		throw new Error(`Plugin "${manifest.id}" from npm package "${packageName}" has already been published under npm package "${originalManifest._npm_package_name}". Plugin from package "${packageName}" will not be imported.`);
 	}
@@ -145,6 +167,7 @@ async function main() {
 	const repoDir = path.resolve(path.dirname(rootDir), 'joplin-plugins');
 	const tempDir = `${repoDir}/temp`;
 	const pluginManifestsPath = path.resolve(repoDir, 'manifests.json');
+	const obsoleteManifestsPath = path.resolve(repoDir, 'obsoletes.json');
 	const errorsPath = path.resolve(repoDir, 'errors.json');
 
 	await checkPluginRepository(repoDir);
@@ -152,6 +175,11 @@ async function main() {
 	await fs.mkdirp(tempDir);
 
 	const originalPluginManifests = await readJsonFile(pluginManifestsPath, {});
+	const obsoleteManifests = await readJsonFile(obsoleteManifestsPath, {});
+	const existingManifests = {
+		...originalPluginManifests,
+		...obsoleteManifests,
+	};
 
 	const searchResults = (await execCommand('npm search joplin-plugin --searchlimit 5000 --json')).trim();
 	const npmPackages = pluginInfoFromSearchResults(JSON.parse(searchResults));
@@ -172,8 +200,8 @@ async function main() {
 		try {
 			const packageName = npmPackage.name;
 			const destDir = `${repoDir}/plugins/`;
-			const manifest = await extractPluginFilesFromPackage(originalPluginManifests, packageTempDir, packageName, destDir);
+			const manifest = await extractPluginFilesFromPackage(existingManifests, packageTempDir, packageName, destDir);
-			manifests[manifest.id] = manifest;
+			if (!obsoleteManifests[manifest.id]) manifests[manifest.id] = manifest;
 		} catch (error) {
 			console.error(error);
 			errors.push(error);