/* eslint-disable import/prefer-default-export */ import { execCommand, getRootDir } from '@joplin/utils'; import { readFile, readdir, stat, writeFile } from 'fs/promises'; import * as MarkdownIt from 'markdown-it'; import { htmlentities, isSelfClosingTag } from '@joplin/utils/html'; import { compileWithFrontMatter, stripOffFrontMatter } from './utils/frontMatter'; import StateCore = require('markdown-it/lib/rules_core/state_core'); import { copy, mkdirp, remove, pathExists } from 'fs-extra'; import { basename, dirname } from 'path'; import markdownUtils, { MarkdownTable } from '@joplin/lib/markdownUtils'; import { readmeFileTitle } from './utils/parser'; import { chdir } from 'process'; import yargs = require('yargs'); import { extractOpenGraphTags } from './utils/openGraph'; const md5File = require('md5-file'); const htmlparser2 = require('@joplin/fork-htmlparser2'); const styleToJs = require('style-to-js').default; const crypto = require('crypto'); interface Config { baseUrl: string; docusaurusBuildEnabled: boolean; docusaurusBuildCommand: string; } interface List { type: 'bullet' | 'number'; } interface Context { currentTable?: MarkdownTable; currentTableContent?: string[]; currentTableCellIndex?: number; inHeader?: boolean; listStack?: List[]; listStarting?: boolean; currentLinkAttrs?: any; inFence?: boolean; processedFiles?: string[]; isNews?: boolean; donateLinks?: string; inQuote?: boolean; } const configs: Record = { dev: { baseUrl: 'http://localhost:8077', docusaurusBuildCommand: 'buildDev', docusaurusBuildEnabled: true, }, prod: { baseUrl: 'https://joplinapp.org', docusaurusBuildCommand: '_build', docusaurusBuildEnabled: true, }, }; const md5 = (s: string) => { return crypto.createHash('md5').update(s).digest('hex'); }; const parseHtml = (html: string) => { const output: string[] = []; interface Tag { name: string; } const tagStack: Tag[] = []; const currentTag = () => { if (!tagStack.length) return { name: '', processed: false }; return tagStack[tagStack.length - 1]; }; const parser = new htmlparser2.Parser({ onopentag: (name: string, attrs: Record) => { tagStack.push({ name }); const closingSign = isSelfClosingTag(name) ? '/>' : '>'; const attrHtml: string[] = []; for (const n in attrs) { if (!attrs.hasOwnProperty(n)) continue; let escapedValue = `"${htmlentities(attrs[n])}"`; if (n === 'style') { escapedValue = `{${JSON.stringify(styleToJs(attrs.style))}}`; } attrHtml.push(`${n}=${escapedValue}`); } output.push(`<${name} ${attrHtml.join(' ')}${closingSign}`); }, ontext: (decodedText: string) => { output.push(htmlentities(decodedText)); }, onclosetag: (name: string) => { const current = currentTag(); if (current.name === name) tagStack.pop(); if (isSelfClosingTag(name)) return; output.push(``); }, }, { decodeEntities: true }); parser.write(html); parser.end(); return output.join(''); }; const escapeForMdx = (s: string): string => { return s .replace(//g, '>') .replace(/`/g, '\\`'); }; const paragraphBreak = '///PARAGRAPH_BREAK///'; const blockQuoteStart = '///BLOCK_QUOTE_START///'; const blockQuoteEnd = '///BLOCK_QUOTE_END///'; const processToken = (token: any, output: string[], context: Context): void => { if (!context.listStack) context.listStack = []; let contentProcessed = false; const type = token.type as string; const currentList = context.listStack.length ? context.listStack[context.listStack.length - 1] : null; const doEscapeForMdx = (text: string) => { if (context.inFence) return text; return escapeForMdx(text); }; const content: string[] = []; if (type === 'heading_open') { content.push(paragraphBreak); content.push(`${token.markup} `); } else if (type === 'heading_close') { content.push(paragraphBreak); } else if (type === 'paragraph_open' || type === 'paragraph_close') { if (!currentList) { content.push(paragraphBreak); } } else if (type === 'fence') { context.inFence = true; content.push(`\`\`\`${token.info || ''}\n`); } else if (type === 'html_block') { contentProcessed = true, content.push(parseHtml(token.content.trim())); } else if (type === 'html_inline') { contentProcessed = true, content.push(parseHtml(token.content.trim())); } else if (type === 'code_inline') { contentProcessed = true, content.push(`\`${token.content}\``); } else if (type === 'code_block') { contentProcessed = true; content.push(`\`\`\`\n${token.content}\`\`\``); content.push(paragraphBreak); } else if (type === 'strong_open' || type === 'strong_close') { content.push('**'); } else if (type === 'em_open' || type === 'em_close') { content.push('*'); } else if (type === 'blockquote_open' || type === 'blockquote_close') { content.push(type === 'blockquote_open' ? blockQuoteStart : blockQuoteEnd); } else if (type === 'table_open') { context.currentTable = { header: [], rows: [], }; context.currentTableContent = []; } else if (type === 'th_open') { context.currentTable.header.push({ label: '', name: `${context.currentTable.header.length}`, disableHtmlEscape: true, }); } else if (type === 'thead_open') { context.inHeader = true; } else if (type === 'thead_close') { context.inHeader = false; } else if (type === 'th_close') { context.currentTable.header[context.currentTable.header.length - 1].label = context.currentTableContent.join(''); context.currentTableContent = []; } else if (type === 'tr_open') { if (!context.inHeader) { context.currentTable.rows.push({}); context.currentTableCellIndex = 0; } } else if (type === 'td_open') { const row = context.currentTable.rows[context.currentTable.rows.length - 1]; row[`${context.currentTableCellIndex}`] = ''; } else if (type === 'td_close') { const row = context.currentTable.rows[context.currentTable.rows.length - 1]; row[`${context.currentTableCellIndex}`] = context.currentTableContent.join(''); context.currentTableCellIndex++; context.currentTableContent = []; } else if (type === 'table_close') { const tableMd = markdownUtils.createMarkdownTable(context.currentTable.header, context.currentTable.rows); content.push(paragraphBreak); content.push(tableMd); content.push(paragraphBreak); context.currentTable = null; } else if (type === 'bullet_list_open') { context.listStarting = !context.listStack.length; context.listStack.push({ type: 'bullet' }); } else if (type === 'ordered_list_open') { context.listStarting = !context.listStack.length; context.listStack.push({ type: 'number' }); } else if (type === 'bullet_list_close' || type === 'ordered_list_close') { context.listStack.pop(); if (!context.listStack.length) content.push(paragraphBreak); } else if (type === 'list_item_open') { if (!context.listStarting) content.push('\n'); context.listStarting = false; const indent = '\t'.repeat((context.listStack.length - 1)); if (currentList.type === 'bullet') content.push(`${indent}- `); if (currentList.type === 'number') content.push(`${indent + token.info}. `); } else if (type === 'image') { (context.currentTable ? context.currentTableContent : content).push('!['); } else if (type === 'link_open') { content.push('['); context.currentLinkAttrs = token.attrs; } else if (type === 'link_close') { const href = context.currentLinkAttrs.find((a: string[]) => a[0] === 'href')[1]; content.push(`](${escapeForMdx(href)})`); context.currentLinkAttrs = null; } if (token.children) { for (const child of token.children) { processToken(child, content, context); } } else if (token.content && !contentProcessed) { content.push(doEscapeForMdx(token.content)); } if (type === 'fence') { content.push('```'); content.push(paragraphBreak); context.inFence = false; } if (type === 'image') { let src: string = token.attrs.find((a: string[]) => a[0] === 'src')[1]; if (!src.startsWith('http') && !src.startsWith('/')) src = `/${src}`; content.push(`](${doEscapeForMdx(src)})`); } for (const c of content) { if (context.currentTable) { context.currentTableContent.push(c); } else { output.push(c); } } }; const resolveParagraphBreaks = (output: string[]): string[] => { if (!output.length) return output; while (output.length && output[0] === paragraphBreak) { output.splice(0, 1); } while (output.length && output[output.length - 1] === paragraphBreak) { output.pop(); } let previous = ''; const newOutput: string[] = []; for (const s of output) { if (s === paragraphBreak && previous === paragraphBreak) continue; newOutput.push(s === paragraphBreak ? '\n\n' : s); previous = s; } return newOutput; }; const resolveBlockQuotes = (output: string): string => { if (!output) return output; const lines = output.split('\n'); let inQuotes = false; const quotedLines: string[] = []; const newOutput: string[] = []; for (const line of lines) { if (line === blockQuoteStart) { inQuotes = true; } else if (line === blockQuoteEnd) { inQuotes = false; const s = quotedLines.join('\n').trim().split('\n').map(l => `> ${l}`).join('\n'); newOutput.push(s); } else if (inQuotes) { quotedLines.push(line); } else { newOutput.push(line); } } return newOutput.join('\n'); }; export const processUrls = (md: string) => { md = md .replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\/index\.md/g, '/help/$1') .replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\.md/g, '/help/$1'); return md; }; export const processMarkdownDoc = (sourceContent: string, context: Context): string => { const markdownIt = new MarkdownIt({ breaks: true, linkify: true, html: true, }); let items: string[] = []; markdownIt.core.ruler.push('converter', (state: StateCore) => { const tokens = state.tokens; // console.info(JSON.stringify(tokens, null, '\t')); for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; processToken(token, items, context); } }); const mdAndFrontMatter = stripOffFrontMatter(sourceContent); markdownIt.render(mdAndFrontMatter.doc); items = resolveParagraphBreaks(items); const output = resolveBlockQuotes(processUrls(items.join('').trim())); return compileWithFrontMatter({ ...mdAndFrontMatter, doc: output, }); }; const processMarkdownFile = async (sourcePath: string, destPath: string, context: Context) => { context.processedFiles.push(destPath); const sourceContent = await readFile(sourcePath, 'utf-8'); const destContent = processMarkdownDoc(sourceContent, context); await mkdirp(dirname(destPath)); const title = await readmeFileTitle(sourcePath); const mdAndFrontMatter = stripOffFrontMatter(destContent); mdAndFrontMatter.header.sidebar_label = title; if (context.isNews) { const dateString = basename(sourcePath).split('-')[0]; const formatted = `${dateString.substring(0, 4)}-${dateString.substring(4, 6)}-${dateString.substring(6, 8)}`; mdAndFrontMatter.header.date = formatted; } const og = await extractOpenGraphTags(sourcePath); mdAndFrontMatter.header.title = og.title; mdAndFrontMatter.header.description = og.description; if (og.image) mdAndFrontMatter.header.image = og.image; mdAndFrontMatter.doc = mdAndFrontMatter.doc.replace(/^# (.*)\n/, `# $1\n\n${context.donateLinks}\n\n`); if (mdAndFrontMatter.header.forum_url) { mdAndFrontMatter.doc += `\n\n* * *\n\n[ Discuss on the forum](${mdAndFrontMatter.header.forum_url})`; } const fullContent = compileWithFrontMatter(mdAndFrontMatter); if (await pathExists(destPath)) { const existingMd5 = await md5File(destPath); const newMd5 = md5(fullContent); if (existingMd5 === newMd5) return; } await writeFile(destPath, fullContent, 'utf-8'); }; const processDocFiles = async (sourceDir: string, destDir: string, excluded: string[], context: Context) => { if (!context.processedFiles) context.processedFiles = []; await mkdirp(destDir); const files = await readdir(sourceDir); for (const file of files) { const fullPath = `${sourceDir}/${file}`; if (excluded.includes(fullPath)) continue; const info = await stat(fullPath); const destPath = `${destDir}/${file}`; if (info.isDirectory()) { await processDocFiles(fullPath, destPath, excluded, context); } else { if (file.endsWith('.md')) { console.info(`Process: ${fullPath}`); await processMarkdownFile(fullPath, destPath, context); } else if (file === '_category_.yml') { context.processedFiles.push(destPath); await copy(fullPath, destPath); } } } }; export const deleteUnprocessedFiles = async (dirPath: string, processedFiles: string[]) => { const files = await readdir(dirPath); for (const file of files) { const fullPath = `${dirPath}/${file}`; const info = await stat(fullPath); if (info.isDirectory()) { await deleteUnprocessedFiles(fullPath, processedFiles); } else { if (!processedFiles.includes(fullPath)) { console.info(`Delete: ${fullPath}`); await remove(fullPath); } } } }; const copyFile = async (sourceFile: string, destFile: string) => { console.info(`Copy: ${destFile}`); await copy(sourceFile, destFile); }; const getDonateLinks = () => { return `
[![Donate using PayPal](https://raw.githubusercontent.com/laurent22/joplin/dev/Assets/WebsiteAssets/images/badges/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?business=E8JMYD2LQ8MMA&no_recurring=0&item_name=I+rely+on+donations+to+maintain+and+improve+the+Joplin+open+source+project.+Thank+you+for+your+help+-+it+makes+a+difference%21¤cy_code=EUR) [![Sponsor on GitHub](https://raw.githubusercontent.com/laurent22/joplin/dev/Assets/WebsiteAssets/images/badges/GitHub-Badge.svg)](https://github.com/sponsors/laurent22/) [![Become a patron](https://raw.githubusercontent.com/laurent22/joplin/dev/Assets/WebsiteAssets/images/badges/Patreon-Badge.svg)](https://www.patreon.com/joplin) [![Donate using IBAN](https://raw.githubusercontent.com/laurent22/joplin/dev/Assets/WebsiteAssets/images/badges/Donate-IBAN.svg)](https://joplinapp.org/donate/#donations)
`; }; export const buildDocusaurus = async (docBuilderDir: string, buildCommand: string, baseUrl: string) => { chdir(docBuilderDir); await execCommand(`yarn ${buildCommand}`, { env: { WEBSITE_BASE_URL: baseUrl, }, }); }; async function main() { const argv = await yargs.argv; const env = argv.env as string; if (!env) throw new Error('Env must be specified: either "dev" or "prod"'); const config = { ...configs[env] }; if ('docuBuild' in argv && !argv.docuBuild) config.docusaurusBuildEnabled = false; const rootDir = await getRootDir(); const docBuilderDir = `${rootDir}/packages/doc-builder`; const destHelpDir = `${docBuilderDir}/help`; const newsDestDir = `${docBuilderDir}/news`; const readmeDir = `${rootDir}/readme`; const donateLinks = getDonateLinks(); const mainContext: Context = { donateLinks }; await processDocFiles(readmeDir, destHelpDir, [ `${readmeDir}/_i18n`, `${readmeDir}/i18n`, `${readmeDir}/cla.md`, `${readmeDir}/download.md`, `${readmeDir}/faq_joplin_cloud.md`, `${readmeDir}/privacy.md`, `${readmeDir}/donate.md`, `${readmeDir}/connection_check.md`, `${readmeDir}/welcome`, `${readmeDir}/news`, ], mainContext); await deleteUnprocessedFiles(destHelpDir, mainContext.processedFiles); const newsContext: Context = { isNews: true, donateLinks }; await processDocFiles(`${readmeDir}/news`, newsDestDir, [], newsContext); await deleteUnprocessedFiles(newsDestDir, newsContext.processedFiles); if (await pathExists(`${readmeDir}/i18n`)) { const localeContext: Context = { donateLinks }; await processDocFiles(`${readmeDir}/i18n`, `${docBuilderDir}/i18n`, [], localeContext); await deleteUnprocessedFiles(`${docBuilderDir}/i18n`, localeContext.processedFiles); } else { console.info('i18n folder is missing - skipping it'); } await copyFile(`${rootDir}/Assets/WebsiteAssets/images`, `${docBuilderDir}/static/images`); if (config.docusaurusBuildEnabled) { await buildDocusaurus(docBuilderDir, config.docusaurusBuildCommand, config.baseUrl); } else { console.info('Skipping Docusaurus build...'); } } if (require.main === module) { // eslint-disable-next-line promise/prefer-await-to-then main().catch((error) => { console.error('Fatal error'); console.error(error); process.exit(1); }); }