diff --git a/.eslintignore b/.eslintignore index acbedd746..292d3b63e 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2023,6 +2023,9 @@ packages/tools/website/build.js.map packages/tools/website/updateDownloadPage.d.ts packages/tools/website/updateDownloadPage.js packages/tools/website/updateDownloadPage.js.map +packages/tools/website/updateNews.d.ts +packages/tools/website/updateNews.js +packages/tools/website/updateNews.js.map packages/tools/website/utils/frontMatter.d.ts packages/tools/website/utils/frontMatter.js packages/tools/website/utils/frontMatter.js.map diff --git a/.gitignore b/.gitignore index 305dcbf09..50543c86c 100644 --- a/.gitignore +++ b/.gitignore @@ -2013,6 +2013,9 @@ packages/tools/website/build.js.map packages/tools/website/updateDownloadPage.d.ts packages/tools/website/updateDownloadPage.js packages/tools/website/updateDownloadPage.js.map +packages/tools/website/updateNews.d.ts +packages/tools/website/updateNews.js +packages/tools/website/updateNews.js.map packages/tools/website/utils/frontMatter.d.ts packages/tools/website/utils/frontMatter.js packages/tools/website/utils/frontMatter.js.map diff --git a/package.json b/package.json index abeea5f7b..74670e449 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "buildCommandIndex": "gulp buildCommandIndex", "buildPluginDoc": "typedoc --name 'Joplin Plugin API Documentation' --mode file -theme './Assets/PluginDocTheme/' --readme './Assets/PluginDocTheme/index.md' --excludeNotExported --excludeExternals --excludePrivate --excludeProtected --out ../joplin-website/docs/api/references/plugin_api packages/lib/services/plugins/api/", "updateMarkdownDoc": "node ./packages/tools/updateMarkdownDoc", + "updateNews": "node ./packages/tools/updateNews", "buildSettingJsonSchema": "yarn workspace joplin start settingschema ../../../joplin-website/docs/schema/settings.json", "buildTranslations": "node packages/tools/build-translation.js", "buildWebsite": "node ./packages/tools/website/build.js && yarn run buildPluginDoc && yarn run buildSettingJsonSchema", diff --git a/packages/tools/package.json b/packages/tools/package.json index 6e7af80ce..c2c8f2240 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -22,6 +22,7 @@ "dependencies": { "@joplin/lib": "~2.7", "@joplin/renderer": "~2.7", + "@types/node-fetch": "1.6.9", "dayjs": "^1.10.7", "execa": "^4.1.0", "fs-extra": "^4.0.3", @@ -31,7 +32,7 @@ "md5-file": "^4.0.0", "moment": "^2.24.0", "mustache": "^2.3.0", - "node-fetch": "^1.7.3", + "node-fetch": "1.7.3", "relative": "^3.0.2", "request": "^2.88.0", "sharp": "^0.25.2", diff --git a/packages/tools/release-website.sh b/packages/tools/release-website.sh index 951202d04..babf768df 100755 --- a/packages/tools/release-website.sh +++ b/packages/tools/release-website.sh @@ -41,6 +41,7 @@ yarn install git reset --hard yarn run updateMarkdownDoc +yarn run updateNews $DISCOURSE_API_KEY $DISCOURSE_USERNAME # We commit and push the change. It will be a noop if nothing was actually # changed diff --git a/packages/tools/website/updateNews.ts b/packages/tools/website/updateNews.ts new file mode 100644 index 000000000..94eb10886 --- /dev/null +++ b/packages/tools/website/updateNews.ts @@ -0,0 +1,190 @@ +// This script reads through the Markdown files in readme/news and post each of +// them as Dicourse forum posts. It then also update the news file with a link +// to that forum post. + +import { readdir, readFile, writeFile } from 'fs-extra'; +import { basename } from 'path'; +import { rootDir } from '../tool-utils'; +import fetch from 'node-fetch'; +import { compileWithFrontMatter, MarkdownAndFrontMatter, stripOffFrontMatter } from './utils/frontMatter'; + +interface ApiConfig { + baseUrl: string; + key: string; + username: string; + newsCategoryId: number; +} + +interface Post { + id: string; + path: string; +} + +interface PostContent { + title: string; + body: string; + parsed: MarkdownAndFrontMatter; +} + +enum HttpMethod { + GET = 'GET', + POST = 'POST', + DELETE = 'DELETE', + PUT = 'PUT', + PATCH = 'PATCH', +} + +interface ForumTopPost { + id: number; + raw: string; + title: string; +} + +const ignoredPostIds = ['20180621-172112','20180621-182112','20180906-101039','20180906-111039','20180916-200431','20180916-210431','20180929-111053','20180929-121053','20181004-081123','20181004-091123','20181101-174335','20181213-173459','20190130-230218','20190404-064157','20190404-074157','20190424-102410','20190424-112410','20190523-221026','20190523-231026','20190610-230711','20190611-000711','20190613-192613','20190613-202613','20190814-215957','20190814-225957','20190924-230254','20190925-000254','20190929-142834','20190929-152834','20191012-223121','20191012-233121','20191014-155136','20191014-165136','20191101-131852','20191117-183855','20191118-072700','20200220-190804','20200301-125055','20200314-001555','20200406-214254','20200406-224254','20200505-181736','20200606-151446','20200607-112720','20200613-103545','20200616-191918','20200620-114515','20200622-084127','20200626-134029','20200708-192444','20200906-172325','20200913-163730','20200915-091108','20201030-114530','20201126-114649','20201130-145937','20201212-172039','20201228-112150','20210104-131645','20210105-153008','20210130-144626','20210309-111950','20210310-100852','20210413-091132','20210430-083248','20210506-083359','20210513-095238','20210518-085514','20210621-104753','20210624-171844','20210705-094247','20210706-140228','20210711-095626','20210718-103538','20210729-103234','20210804-085003','20210831-154354','20210901-113415','20210929-144036','20210930-163458','20211031-115215','20211102-150403','20211217-120324','20220215-142000','20220224-release-2-7','20220308-gsoc2022-start']; + +const config: ApiConfig = { + baseUrl: 'https://discourse.joplinapp.org', + key: '', + username: '', + newsCategoryId: 9, +}; + +const getPosts = async (newsDir: string): Promise => { + const filenames = await readdir(newsDir); + const output: Post[] = []; + + for (const filename of filenames) { + if (!filename.endsWith('.md')) continue; + output.push({ + id: basename(filename, '.md'), + path: `${newsDir}/${filename}`, + }); + } + + return output; +}; + +const getPostContent = async (post: Post): Promise => { + const raw = await readFile(post.path, 'utf8'); + const parsed = stripOffFrontMatter(raw); + const lines = parsed.doc.split('\n'); + const titleLine = lines[0]; + if (!titleLine.startsWith('# ')) throw new Error('Cannot extract title from post: no header detected'); + lines.splice(0, 1); + + return { + title: titleLine.substr(1).trim(), + body: lines.join('\n').trim(), + parsed, + }; +}; + +const execApi = async (method: HttpMethod, path: string, body: Record = null) => { + const headers: Record = { + 'Api-Key': config.key, + 'Api-Username': config.username, + }; + + if (method !== HttpMethod.GET) headers['Content-Type'] = 'application/json;'; + + const response = await fetch(`${config.baseUrl}/${path}`, { + method, + headers, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const errorText = await response.text(); + const error = new Error(`On ${method} ${path}: ${errorText}`); + let apiObject = null; + try { + apiObject = JSON.parse(errorText); + } catch (error) { + // Ignore - it just means that the error object is a plain string + } + (error as any).apiObject = apiObject; + throw error; + } + + return response.json(); +}; + +const getForumTopPostByExternalId = async (externalId: string): Promise => { + try { + const existingForumTopic = await execApi(HttpMethod.GET, `t/external_id/${externalId}.json`); + const existingForumPost = await execApi(HttpMethod.GET, `posts/${existingForumTopic.post_stream.posts[0].id}.json`); + return { + id: existingForumPost.id, + title: existingForumTopic.title, + raw: existingForumPost.raw, + }; + } catch (error) { + if (error.apiObject && error.apiObject.error_type === 'not_found') return null; + throw error; + } +}; + +const main = async () => { + const argv = require('yargs').argv; + config.key = argv._[0]; + config.username = argv._[1]; + + if (!config.key || !config.username) throw new Error('API Key and Username are required'); + + const posts = await getPosts(`${rootDir}/readme/news`); + + for (const post of posts) { + if (ignoredPostIds.includes(post.id)) continue; + + console.info(`Processing ${post.path}...`); + + try { + const content = await getPostContent(post); + const existingForumPost = await getForumTopPostByExternalId(post.id); + + if (existingForumPost) { + // console.info('EXISTING ========================'); + // console.info(existingForumPost.title); + // console.info(existingForumPost.raw); + + // console.info('NEW ========================'); + // console.info(content.title); + // console.info(content.body); + + if (existingForumPost.title === content.title && existingForumPost.raw === content.body) { + console.info('Post already exists and has not changed: skipping it...'); + } else { + console.info('Post already exists and has changed: updating it...'); + + await execApi(HttpMethod.PUT, `posts/${existingForumPost.id}.json`, { + title: content.title, + raw: content.body, + edit_reason: 'Auto-updated by script', + }); + } + } else { + console.info('Post does not exists: creating it...'); + + const response = await execApi(HttpMethod.POST, 'posts', { + title: content.title, + raw: content.body, + category: config.newsCategoryId, + external_id: post.id, + }); + + const postUrl = `https://discourse.joplinapp.org/t/${response.topic_id}`; + content.parsed.forum_url = postUrl; + const compiled = compileWithFrontMatter(content.parsed); + + await writeFile(post.path, compiled, 'utf8'); + } + } catch (error) { + console.error(error); + } + } +}; + +main().catch((error) => { + console.error('Fatal error', error); + process.exit(1); +}); diff --git a/packages/tools/website/utils/frontMatter.ts b/packages/tools/website/utils/frontMatter.ts index f5ac06729..19b2e5e4c 100644 --- a/packages/tools/website/utils/frontMatter.ts +++ b/packages/tools/website/utils/frontMatter.ts @@ -5,6 +5,7 @@ export interface MarkdownAndFrontMatter { created?: Date; updated?: Date; source_url?: string; + forum_url?: string; } const readProp = (line: string): string[] => { @@ -62,3 +63,37 @@ export const stripOffFrontMatter = (md: string): MarkdownAndFrontMatter => { return output; }; + +// --- +// created: 2021-07-05T09:42:47.000+00:00 +// source_url: https://www.patreon.com/posts/any-ideas-for-53317699 +// --- + +const formatFrontMatterValue = (key: string, value: any) => { + if (['created', 'updated'].includes(key)) { + return moment((value as Date)).toISOString(); + } else { + return value.toString(); + } +}; + +export const compileWithFrontMatter = (md: MarkdownAndFrontMatter): string => { + const output: string[] = []; + const header: string[] = []; + + for (const [key, value] of Object.entries(md)) { + if (key === 'doc') continue; + header.push(`${key}: ${formatFrontMatterValue(key, value)}`); + } + + if (header.length) { + output.push('---'); + output.push(header.join('\n')); + output.push('---'); + output.push(''); + } + + output.push(md.doc); + + return output.join('\n'); +}; diff --git a/readme/news/20220405-gsoc-contributor-proposals.md b/readme/news/20220405-gsoc-contributor-proposals.md index 48c0b821f..c6270e44e 100644 --- a/readme/news/20220405-gsoc-contributor-proposals.md +++ b/readme/news/20220405-gsoc-contributor-proposals.md @@ -1,3 +1,7 @@ +--- +forum_url: https://discourse.joplinapp.org/t/24913 +--- + # GSoC "Contributor Proposals" phase is starting now! The "Contributor Proposals" phase of GSoC 2022 is starting today! If you would like to be a contributor, now is the time to choose your project idea, write your proposal, and upload it to https://summerofcode.withgoogle.com/ diff --git a/yarn.lock b/yarn.lock index 8d67c2a64..0a26d038b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3444,6 +3444,7 @@ __metadata: "@types/jest": ^26.0.15 "@types/mustache": ^0.8.32 "@types/node": ^14.14.6 + "@types/node-fetch": 1.6.9 dayjs: ^1.10.7 execa: ^4.1.0 fs-extra: ^4.0.3 @@ -3456,7 +3457,7 @@ __metadata: md5-file: ^4.0.0 moment: ^2.24.0 mustache: ^2.3.0 - node-fetch: ^1.7.3 + node-fetch: 1.7.3 relative: ^3.0.2 request: ^2.88.0 sass: ^1.39.2 @@ -5575,6 +5576,15 @@ __metadata: languageName: node linkType: hard +"@types/node-fetch@npm:1.6.9": + version: 1.6.9 + resolution: "@types/node-fetch@npm:1.6.9" + dependencies: + "@types/node": "*" + checksum: 9c5306e852275a464ec6106d8ce7f0b7d009b2a62a613ea183601e3c469cc66d6c22fe786d12bdebbdece396df3b53cc387a32b2834634cd5f4b8b2d4fe1136c + languageName: node + linkType: hard + "@types/node-rsa@npm:^1.1.1": version: 1.1.1 resolution: "@types/node-rsa@npm:1.1.1" @@ -22049,7 +22059,7 @@ __metadata: languageName: node linkType: hard -"node-fetch@npm:^1.0.1, node-fetch@npm:^1.7.1, node-fetch@npm:^1.7.3": +"node-fetch@npm:1.7.3, node-fetch@npm:^1.0.1, node-fetch@npm:^1.7.1, node-fetch@npm:^1.7.3": version: 1.7.3 resolution: "node-fetch@npm:1.7.3" dependencies: