// This script reads through the Markdown files in readme/news and post each of // them as Dicourse forum posts. It then also update the news file with a link // to that forum post. import { readdir, readFile, writeFile } from 'fs-extra'; import { basename } from 'path'; import { rootDir } from '../tool-utils'; import fetch from 'node-fetch'; import { compileWithFrontMatter, MarkdownAndFrontMatter, stripOffFrontMatter } from './utils/frontMatter'; interface ApiConfig { baseUrl: string; key: string; username: string; newsCategoryId: number; } interface Post { id: string; path: string; } interface PostContent { title: string; body: string; parsed: MarkdownAndFrontMatter; } enum HttpMethod { GET = 'GET', POST = 'POST', DELETE = 'DELETE', PUT = 'PUT', PATCH = 'PATCH', } interface ForumTopPost { id: number; raw: string; title: string; } const ignoredPostIds = ['20180621-172112','20180621-182112','20180906-101039','20180906-111039','20180916-200431','20180916-210431','20180929-111053','20180929-121053','20181004-081123','20181004-091123','20181101-174335','20181213-173459','20190130-230218','20190404-064157','20190404-074157','20190424-102410','20190424-112410','20190523-221026','20190523-231026','20190610-230711','20190611-000711','20190613-192613','20190613-202613','20190814-215957','20190814-225957','20190924-230254','20190925-000254','20190929-142834','20190929-152834','20191012-223121','20191012-233121','20191014-155136','20191014-165136','20191101-131852','20191117-183855','20191118-072700','20200220-190804','20200301-125055','20200314-001555','20200406-214254','20200406-224254','20200505-181736','20200606-151446','20200607-112720','20200613-103545','20200616-191918','20200620-114515','20200622-084127','20200626-134029','20200708-192444','20200906-172325','20200913-163730','20200915-091108','20201030-114530','20201126-114649','20201130-145937','20201212-172039','20201228-112150','20210104-131645','20210105-153008','20210130-144626','20210309-111950','20210310-100852','20210413-091132','20210430-083248','20210506-083359','20210513-095238','20210518-085514','20210621-104753','20210624-171844','20210705-094247','20210706-140228','20210711-095626','20210718-103538','20210729-103234','20210804-085003','20210831-154354','20210901-113415','20210929-144036','20210930-163458','20211031-115215','20211102-150403','20211217-120324','20220215-142000','20220224-release-2-7','20220308-gsoc2022-start','20220405-gsoc-contributor-proposals']; const config: ApiConfig = { baseUrl: 'https://discourse.joplinapp.org', key: '', username: '', newsCategoryId: 9, }; const getPosts = async (newsDir: string): Promise => { const filenames = await readdir(newsDir); const output: Post[] = []; for (const filename of filenames) { if (!filename.endsWith('.md')) continue; output.push({ id: basename(filename, '.md'), path: `${newsDir}/${filename}`, }); } return output; }; const getPostContent = async (post: Post): Promise => { const raw = await readFile(post.path, 'utf8'); const parsed = stripOffFrontMatter(raw); const lines = parsed.doc.split('\n'); const titleLine = lines[0]; if (!titleLine.startsWith('# ')) throw new Error('Cannot extract title from post: no header detected'); lines.splice(0, 1); return { title: titleLine.substr(1).trim(), body: lines.join('\n').trim(), parsed, }; }; const execApi = async (method: HttpMethod, path: string, body: Record = null) => { const headers: Record = { 'Api-Key': config.key, 'Api-Username': config.username, }; if (method !== HttpMethod.GET) headers['Content-Type'] = 'application/json;'; const response = await fetch(`${config.baseUrl}/${path}`, { method, headers, body: JSON.stringify(body), }); if (!response.ok) { const errorText = await response.text(); const error = new Error(`On ${method} ${path}: ${errorText}`); let apiObject = null; try { apiObject = JSON.parse(errorText); } catch (error) { // Ignore - it just means that the error object is a plain string } (error as any).apiObject = apiObject; throw error; } return response.json(); }; const getForumTopPostByExternalId = async (externalId: string): Promise => { try { const existingForumTopic = await execApi(HttpMethod.GET, `t/external_id/${externalId}.json`); const existingForumPost = await execApi(HttpMethod.GET, `posts/${existingForumTopic.post_stream.posts[0].id}.json`); return { id: existingForumPost.id, title: existingForumTopic.title, raw: existingForumPost.raw, }; } catch (error) { if (error.apiObject && error.apiObject.error_type === 'not_found') return null; throw error; } }; const main = async () => { const argv = require('yargs').argv; config.key = argv._[0]; config.username = argv._[1]; if (!config.key || !config.username) throw new Error('API Key and Username are required'); const posts = await getPosts(`${rootDir}/readme/news`); for (const post of posts) { if (ignoredPostIds.includes(post.id)) continue; console.info(`Processing ${post.path}...`); try { const content = await getPostContent(post); const existingForumPost = await getForumTopPostByExternalId(post.id); if (existingForumPost) { // console.info('EXISTING ========================'); // console.info(existingForumPost.title); // console.info(existingForumPost.raw); // console.info('NEW ========================'); // console.info(content.title); // console.info(content.body); if (existingForumPost.title === content.title && existingForumPost.raw === content.body) { console.info('Post already exists and has not changed: skipping it...'); } else { console.info('Post already exists and has changed: updating it...'); await execApi(HttpMethod.PUT, `posts/${existingForumPost.id}.json`, { title: content.title, raw: content.body, edit_reason: 'Auto-updated by script', }); } } else { console.info('Post does not exists: creating it...'); const response = await execApi(HttpMethod.POST, 'posts', { title: content.title, raw: content.body, category: config.newsCategoryId, external_id: post.id, }); const postUrl = `https://discourse.joplinapp.org/t/${response.topic_id}`; content.parsed.forum_url = postUrl; const compiled = compileWithFrontMatter(content.parsed); await writeFile(post.path, compiled, 'utf8'); } } catch (error) { console.error(error); } } }; main().catch((error) => { console.error('Fatal error', error); process.exit(1); });