Tools: Added tool to automatically post news from local Markdown folder to forum

.eslintignore

@@ -2023,6 +2023,9 @@ packages/tools/website/build.js.map
 packages/tools/website/updateDownloadPage.d.ts
 packages/tools/website/updateDownloadPage.js
 packages/tools/website/updateDownloadPage.js.map
+packages/tools/website/updateNews.d.ts
+packages/tools/website/updateNews.js
+packages/tools/website/updateNews.js.map
 packages/tools/website/utils/frontMatter.d.ts
 packages/tools/website/utils/frontMatter.js
 packages/tools/website/utils/frontMatter.js.map

.gitignore

@@ -2013,6 +2013,9 @@ packages/tools/website/build.js.map
 packages/tools/website/updateDownloadPage.d.ts
 packages/tools/website/updateDownloadPage.js
 packages/tools/website/updateDownloadPage.js.map
+packages/tools/website/updateNews.d.ts
+packages/tools/website/updateNews.js
+packages/tools/website/updateNews.js.map
 packages/tools/website/utils/frontMatter.d.ts
 packages/tools/website/utils/frontMatter.js
 packages/tools/website/utils/frontMatter.js.map

package.json

@@ -18,6 +18,7 @@
     "buildCommandIndex": "gulp buildCommandIndex",
     "buildPluginDoc": "typedoc --name 'Joplin Plugin API Documentation' --mode file -theme './Assets/PluginDocTheme/' --readme './Assets/PluginDocTheme/index.md' --excludeNotExported --excludeExternals --excludePrivate --excludeProtected --out ../joplin-website/docs/api/references/plugin_api packages/lib/services/plugins/api/",
     "updateMarkdownDoc": "node ./packages/tools/updateMarkdownDoc",
+    "updateNews": "node ./packages/tools/updateNews",
     "buildSettingJsonSchema": "yarn workspace joplin start settingschema ../../../joplin-website/docs/schema/settings.json",
     "buildTranslations": "node packages/tools/build-translation.js",
     "buildWebsite": "node ./packages/tools/website/build.js && yarn run buildPluginDoc && yarn run buildSettingJsonSchema",

packages/tools/package.json

@@ -22,6 +22,7 @@
   "dependencies": {
     "@joplin/lib": "~2.7",
     "@joplin/renderer": "~2.7",
+    "@types/node-fetch": "1.6.9",
     "dayjs": "^1.10.7",
     "execa": "^4.1.0",
     "fs-extra": "^4.0.3",
@@ -31,7 +32,7 @@
     "md5-file": "^4.0.0",
     "moment": "^2.24.0",
     "mustache": "^2.3.0",
-    "node-fetch": "^1.7.3",
+    "node-fetch": "1.7.3",
     "relative": "^3.0.2",
     "request": "^2.88.0",
     "sharp": "^0.25.2",

packages/tools/release-website.sh

@@ -41,6 +41,7 @@ yarn install
 git reset --hard
 
 yarn run updateMarkdownDoc
+yarn run updateNews $DISCOURSE_API_KEY $DISCOURSE_USERNAME
 
 # We commit and push the change. It will be a noop if nothing was actually
 # changed

packages/tools/website/updateNews.ts

@@ -0,0 +1,190 @@
+// This script reads through the Markdown files in readme/news and post each of
+// them as Dicourse forum posts. It then also update the news file with a link
+// to that forum post.
+
+import { readdir, readFile, writeFile } from 'fs-extra';
+import { basename } from 'path';
+import { rootDir } from '../tool-utils';
+import fetch from 'node-fetch';
+import { compileWithFrontMatter, MarkdownAndFrontMatter, stripOffFrontMatter } from './utils/frontMatter';
+
+interface ApiConfig {
+	baseUrl: string;
+	key: string;
+	username: string;
+	newsCategoryId: number;
+}
+
+interface Post {
+	id: string;
+	path: string;
+}
+
+interface PostContent {
+	title: string;
+	body: string;
+	parsed: MarkdownAndFrontMatter;
+}
+
+enum HttpMethod {
+	GET = 'GET',
+	POST = 'POST',
+	DELETE = 'DELETE',
+	PUT = 'PUT',
+	PATCH = 'PATCH',
+}
+
+interface ForumTopPost {
+	id: number;
+	raw: string;
+	title: string;
+}
+
+const ignoredPostIds = ['20180621-172112','20180621-182112','20180906-101039','20180906-111039','20180916-200431','20180916-210431','20180929-111053','20180929-121053','20181004-081123','20181004-091123','20181101-174335','20181213-173459','20190130-230218','20190404-064157','20190404-074157','20190424-102410','20190424-112410','20190523-221026','20190523-231026','20190610-230711','20190611-000711','20190613-192613','20190613-202613','20190814-215957','20190814-225957','20190924-230254','20190925-000254','20190929-142834','20190929-152834','20191012-223121','20191012-233121','20191014-155136','20191014-165136','20191101-131852','20191117-183855','20191118-072700','20200220-190804','20200301-125055','20200314-001555','20200406-214254','20200406-224254','20200505-181736','20200606-151446','20200607-112720','20200613-103545','20200616-191918','20200620-114515','20200622-084127','20200626-134029','20200708-192444','20200906-172325','20200913-163730','20200915-091108','20201030-114530','20201126-114649','20201130-145937','20201212-172039','20201228-112150','20210104-131645','20210105-153008','20210130-144626','20210309-111950','20210310-100852','20210413-091132','20210430-083248','20210506-083359','20210513-095238','20210518-085514','20210621-104753','20210624-171844','20210705-094247','20210706-140228','20210711-095626','20210718-103538','20210729-103234','20210804-085003','20210831-154354','20210901-113415','20210929-144036','20210930-163458','20211031-115215','20211102-150403','20211217-120324','20220215-142000','20220224-release-2-7','20220308-gsoc2022-start'];
+
+const config: ApiConfig = {
+	baseUrl: 'https://discourse.joplinapp.org',
+	key: '',
+	username: '',
+	newsCategoryId: 9,
+};
+
+const getPosts = async (newsDir: string): Promise<Post[]> => {
+	const filenames = await readdir(newsDir);
+	const output: Post[] = [];
+
+	for (const filename of filenames) {
+		if (!filename.endsWith('.md')) continue;
+		output.push({
+			id: basename(filename, '.md'),
+			path: `${newsDir}/${filename}`,
+		});
+	}
+
+	return output;
+};
+
+const getPostContent = async (post: Post): Promise<PostContent> => {
+	const raw = await readFile(post.path, 'utf8');
+	const parsed = stripOffFrontMatter(raw);
+	const lines = parsed.doc.split('\n');
+	const titleLine = lines[0];
+	if (!titleLine.startsWith('# ')) throw new Error('Cannot extract title from post: no header detected');
+	lines.splice(0, 1);
+
+	return {
+		title: titleLine.substr(1).trim(),
+		body: lines.join('\n').trim(),
+		parsed,
+	};
+};
+
+const execApi = async (method: HttpMethod, path: string, body: Record<string, string | number> = null) => {
+	const headers: Record<string, string> = {
+		'Api-Key': config.key,
+		'Api-Username': config.username,
+	};
+
+	if (method !== HttpMethod.GET) headers['Content-Type'] = 'application/json;';
+
+	const response = await fetch(`${config.baseUrl}/${path}`, {
+		method,
+		headers,
+		body: JSON.stringify(body),
+	});
+
+	if (!response.ok) {
+		const errorText = await response.text();
+		const error = new Error(`On ${method} ${path}: ${errorText}`);
+		let apiObject = null;
+		try {
+			apiObject = JSON.parse(errorText);
+		} catch (error) {
+			// Ignore - it just means that the error object is a plain string
+		}
+		(error as any).apiObject = apiObject;
+		throw error;
+	}
+
+	return response.json();
+};
+
+const getForumTopPostByExternalId = async (externalId: string): Promise<ForumTopPost> => {
+	try {
+		const existingForumTopic = await execApi(HttpMethod.GET, `t/external_id/${externalId}.json`);
+		const existingForumPost = await execApi(HttpMethod.GET, `posts/${existingForumTopic.post_stream.posts[0].id}.json`);
+		return {
+			id: existingForumPost.id,
+			title: existingForumTopic.title,
+			raw: existingForumPost.raw,
+		};
+	} catch (error) {
+		if (error.apiObject && error.apiObject.error_type === 'not_found') return null;
+		throw error;
+	}
+};
+
+const main = async () => {
+	const argv = require('yargs').argv;
+	config.key = argv._[0];
+	config.username = argv._[1];
+
+	if (!config.key || !config.username) throw new Error('API Key and Username are required');
+
+	const posts = await getPosts(`${rootDir}/readme/news`);
+
+	for (const post of posts) {
+		if (ignoredPostIds.includes(post.id)) continue;
+
+		console.info(`Processing ${post.path}...`);
+
+		try {
+			const content = await getPostContent(post);
+			const existingForumPost = await getForumTopPostByExternalId(post.id);
+
+			if (existingForumPost) {
+				// console.info('EXISTING ========================');
+				// console.info(existingForumPost.title);
+				// console.info(existingForumPost.raw);
+
+				// console.info('NEW ========================');
+				// console.info(content.title);
+				// console.info(content.body);
+
+				if (existingForumPost.title === content.title && existingForumPost.raw === content.body) {
+					console.info('Post already exists and has not changed: skipping it...');
+				} else {
+					console.info('Post already exists and has changed: updating it...');
+
+					await execApi(HttpMethod.PUT, `posts/${existingForumPost.id}.json`, {
+						title: content.title,
+						raw: content.body,
+						edit_reason: 'Auto-updated by script',
+					});
+				}
+			} else {
+				console.info('Post does not exists: creating it...');
+
+				const response = await execApi(HttpMethod.POST, 'posts', {
+					title: content.title,
+					raw: content.body,
+					category: config.newsCategoryId,
+					external_id: post.id,
+				});
+
+				const postUrl = `https://discourse.joplinapp.org/t/${response.topic_id}`;
+				content.parsed.forum_url = postUrl;
+				const compiled = compileWithFrontMatter(content.parsed);
+
+				await writeFile(post.path, compiled, 'utf8');
+			}
+		} catch (error) {
+			console.error(error);
+		}
+	}
+};
+
+main().catch((error) => {
+	console.error('Fatal error', error);
+	process.exit(1);
+});

packages/tools/website/utils/frontMatter.ts

@@ -5,6 +5,7 @@ export interface MarkdownAndFrontMatter {
 	created?: Date;
 	updated?: Date;
 	source_url?: string;
+	forum_url?: string;
 }
 
 const readProp = (line: string): string[] => {
@@ -62,3 +63,37 @@ export const stripOffFrontMatter = (md: string): MarkdownAndFrontMatter => {
 
 	return output;
 };
+
+// ---
+// created: 2021-07-05T09:42:47.000+00:00
+// source_url: https://www.patreon.com/posts/any-ideas-for-53317699
+// ---
+
+const formatFrontMatterValue = (key: string, value: any) => {
+	if (['created', 'updated'].includes(key)) {
+		return moment((value as Date)).toISOString();
+	} else {
+		return value.toString();
+	}
+};
+
+export const compileWithFrontMatter = (md: MarkdownAndFrontMatter): string => {
+	const output: string[] = [];
+	const header: string[] = [];
+
+	for (const [key, value] of Object.entries(md)) {
+		if (key === 'doc') continue;
+		header.push(`${key}: ${formatFrontMatterValue(key, value)}`);
+	}
+
+	if (header.length) {
+		output.push('---');
+		output.push(header.join('\n'));
+		output.push('---');
+		output.push('');
+	}
+
+	output.push(md.doc);
+
+	return output.join('\n');
+};

readme/news/20220405-gsoc-contributor-proposals.md

@@ -1,3 +1,7 @@
+---
+forum_url: https://discourse.joplinapp.org/t/24913
+---
+
 # GSoC "Contributor Proposals" phase is starting now!
 
 The "Contributor Proposals" phase of GSoC 2022 is starting today! If you would like to be a contributor, now is the time to choose your project idea, write your proposal, and upload it to https://summerofcode.withgoogle.com/

yarn.lock

@@ -3444,6 +3444,7 @@ __metadata:
     "@types/jest": ^26.0.15
     "@types/mustache": ^0.8.32
     "@types/node": ^14.14.6
+    "@types/node-fetch": 1.6.9
     dayjs: ^1.10.7
     execa: ^4.1.0
     fs-extra: ^4.0.3
@@ -3456,7 +3457,7 @@ __metadata:
     md5-file: ^4.0.0
     moment: ^2.24.0
     mustache: ^2.3.0
-    node-fetch: ^1.7.3
+    node-fetch: 1.7.3
     relative: ^3.0.2
     request: ^2.88.0
     sass: ^1.39.2
@@ -5575,6 +5576,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/node-fetch@npm:1.6.9":
+  version: 1.6.9
+  resolution: "@types/node-fetch@npm:1.6.9"
+  dependencies:
+    "@types/node": "*"
+  checksum: 9c5306e852275a464ec6106d8ce7f0b7d009b2a62a613ea183601e3c469cc66d6c22fe786d12bdebbdece396df3b53cc387a32b2834634cd5f4b8b2d4fe1136c
+  languageName: node
+  linkType: hard
+
 "@types/node-rsa@npm:^1.1.1":
   version: 1.1.1
   resolution: "@types/node-rsa@npm:1.1.1"
@@ -22049,7 +22059,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"node-fetch@npm:^1.0.1, node-fetch@npm:^1.7.1, node-fetch@npm:^1.7.3":
+"node-fetch@npm:1.7.3, node-fetch@npm:^1.0.1, node-fetch@npm:^1.7.1, node-fetch@npm:^1.7.3":
   version: 1.7.3
   resolution: "node-fetch@npm:1.7.3"
   dependencies: