diff --git a/.eslintignore b/.eslintignore index 1b995bd9d..26b39197a 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2011,6 +2011,15 @@ packages/tools/website/updateDownloadPage.js.map packages/tools/website/utils/frontMatter.d.ts packages/tools/website/utils/frontMatter.js packages/tools/website/utils/frontMatter.js.map +packages/tools/website/utils/openGraph.d.ts +packages/tools/website/utils/openGraph.js +packages/tools/website/utils/openGraph.js.map +packages/tools/website/utils/openGraph.test.d.ts +packages/tools/website/utils/openGraph.test.js +packages/tools/website/utils/openGraph.test.js.map +packages/tools/website/utils/parser.d.ts +packages/tools/website/utils/parser.js +packages/tools/website/utils/parser.js.map packages/tools/website/utils/pressCarousel.d.ts packages/tools/website/utils/pressCarousel.js packages/tools/website/utils/pressCarousel.js.map diff --git a/.gitignore b/.gitignore index c88b51fc5..7a000a87c 100644 --- a/.gitignore +++ b/.gitignore @@ -2001,6 +2001,15 @@ packages/tools/website/updateDownloadPage.js.map packages/tools/website/utils/frontMatter.d.ts packages/tools/website/utils/frontMatter.js packages/tools/website/utils/frontMatter.js.map +packages/tools/website/utils/openGraph.d.ts +packages/tools/website/utils/openGraph.js +packages/tools/website/utils/openGraph.js.map +packages/tools/website/utils/openGraph.test.d.ts +packages/tools/website/utils/openGraph.test.js +packages/tools/website/utils/openGraph.test.js.map +packages/tools/website/utils/parser.d.ts +packages/tools/website/utils/parser.js +packages/tools/website/utils/parser.js.map packages/tools/website/utils/pressCarousel.d.ts packages/tools/website/utils/pressCarousel.js packages/tools/website/utils/pressCarousel.js.map diff --git a/Assets/WebsiteAssets/templates/front.mustache b/Assets/WebsiteAssets/templates/front.mustache index 67ac448a8..91a41bacc 100644 --- a/Assets/WebsiteAssets/templates/front.mustache +++ b/Assets/WebsiteAssets/templates/front.mustache @@ -10,8 +10,8 @@ - + {{> openGraphTags}} - + {{> openGraphTags}} + + + + + + {{#openGraph.image}} + + {{/openGraph.image}} +{{/openGraph}} diff --git a/packages/lib/markupLanguageUtils.ts b/packages/lib/markupLanguageUtils.ts index 22f56177f..55f2409c7 100644 --- a/packages/lib/markupLanguageUtils.ts +++ b/packages/lib/markupLanguageUtils.ts @@ -15,8 +15,17 @@ export class MarkupLanguageUtils { throw new Error(`Unsupported markup language: ${language}`); } - public extractImageUrls(language: MarkupLanguage, text: string) { - return this.lib_(language).extractImageUrls(text); + public extractImageUrls(language: MarkupLanguage, text: string): string[] { + let urls: string[] = []; + + if (language === MarkupLanguage.Any) { + urls = urls.concat(this.lib_(MarkupLanguage.Markdown).extractImageUrls(text)); + urls = urls.concat(this.lib_(MarkupLanguage.Html).extractImageUrls(text)); + } else { + urls = this.lib_(language).extractImageUrls(text); + } + + return urls; } // Create a new MarkupToHtml instance while injecting options specific to Joplin diff --git a/packages/renderer/MarkupToHtml.ts b/packages/renderer/MarkupToHtml.ts index e415a760e..0e476c2b5 100644 --- a/packages/renderer/MarkupToHtml.ts +++ b/packages/renderer/MarkupToHtml.ts @@ -7,6 +7,7 @@ const MarkdownIt = require('markdown-it'); export enum MarkupLanguage { Markdown = 1, Html = 2, + Any = 3, } export interface RenderResultPluginAsset { diff --git a/packages/tools/website/build.ts b/packages/tools/website/build.ts index c9a5bfbd9..e5e70e40c 100644 --- a/packages/tools/website/build.ts +++ b/packages/tools/website/build.ts @@ -6,6 +6,8 @@ import { AssetUrls, Env, PlanPageParams, Sponsors, TemplateParams } from './util import { getPlans, loadStripeConfig } from '@joplin/lib/utils/joplinCloud'; import { stripOffFrontMatter } from './utils/frontMatter'; import { dirname, basename } from 'path'; +import { readmeFileTitle, replaceGitHubByWebsiteLinks } from './utils/parser'; +import { extractOpenGraphTags } from './utils/openGraph'; const moment = require('moment'); const glob = require('glob'); @@ -49,14 +51,6 @@ async function getDonateLinks() { return ``; } -function replaceGitHubByWebsiteLinks(md: string) { - return md - .replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\/index\.md(#[^\s)]+|)/g, '/$1/$2') - .replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\.md(#[^\s)]+|)/g, '/$1/$2') - .replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/README\.md(#[^\s)]+|)/g, '/help/$1') - .replace(/https:\/\/raw.githubusercontent.com\/laurent22\/joplin\/dev\/Assets\/WebsiteAssets\/(.*?)/g, '/$1'); -} - function tocHtml() { if (tocHtml_) return tocHtml_; const markdownIt = getMarkdownIt(); @@ -104,6 +98,7 @@ function defaultTemplateParams(assetUrls: AssetUrls): TemplateParams { isFrontPage: false, }, assetUrls, + openGraph: null, }; } @@ -142,18 +137,6 @@ function renderPageToHtml(md: string, targetPath: string, templateParams: Templa writeFileSync(targetPath, html); } -async function readmeFileTitle(sourcePath: string) { - let md = await readFile(sourcePath, 'utf8'); - md = stripOffFrontMatter(md).doc; - const r = md.match(/(^|\n)# (.*)/); - - if (!r) { - throw new Error(`Could not determine title for Markdown file: ${sourcePath}`); - } else { - return r[2]; - } -} - function renderFileToHtml(sourcePath: string, targetPath: string, templateParams: TemplateParams) { let md = readFileSync(sourcePath, 'utf8'); md = stripOffFrontMatter(md).doc; @@ -229,6 +212,11 @@ async function main() { partials, sponsors, assetUrls, + openGraph: { + title: 'Joplin documentation', + description: '', + url: 'https://joplinapp.org/help/', + }, }); // ============================================================= @@ -252,6 +240,11 @@ async function main() { }, showToc: false, assetUrls, + openGraph: { + title: 'Joplin website', + description: 'Joplin, the open source note-taking application', + url: 'https://joplinapp.org', + }, }); // ============================================================= @@ -290,17 +283,17 @@ async function main() { const mdFiles = glob.sync(`${readmeDir}/**/*.md`).map((f: string) => f.substr(rootDir.length + 1)); const sources = []; - const makeTargetFilePath = (input: string): string => { + const makeTargetBasename = (input: string): string => { if (isNewsFile(input)) { const filenameNoExt = basename(input, '.md'); - return `${docDir}/news/${filenameNoExt}/index.html`; + return `news/${filenameNoExt}/index.html`; } else { // Input is for example "readme/spec/interop_with_frontmatter.md", // and we need to convert it to // "docs/spec/interop_with_frontmatter/index.html" and prefix it // with the website repo full path. - let s = `${docDir}/${input}`; + let s = input; if (s.endsWith('index.md')) { s = s.replace(/index\.md/, 'index.html'); } else { @@ -313,11 +306,20 @@ async function main() { } }; + const makeTargetFilePath = (input: string): string => { + return `${docDir}/${makeTargetBasename(input)}`; + }; + + const makeTargetUrl = (input: string) => { + return `https://joplinapp.org/${makeTargetBasename(input)}`; + }; + const newsFilePaths: string[] = []; for (const mdFile of mdFiles) { const title = await readmeFileTitle(`${rootDir}/${mdFile}`); const targetFilePath = makeTargetFilePath(mdFile); + const openGraph = await extractOpenGraphTags(mdFile, makeTargetUrl(mdFile)); const isNews = isNewsFile(mdFile); if (isNews) newsFilePaths.push(mdFile); @@ -326,6 +328,7 @@ async function main() { title: title, donateLinksMd: mdFile === 'readme/donate.md' ? '' : donateLinksMd, showToc: mdFile !== 'readme/download.md' && !isNews, + openGraph, }]); } @@ -354,11 +357,17 @@ async function main() { await makeNewsFrontPage(newsFilePaths, `${docDir}/news/index.html`, { ...defaultTemplateParams(assetUrls), + title: 'What\'s new', pageName: 'news', partials, showToc: false, showImproveThisDoc: false, donateLinksMd, + openGraph: { + title: 'Joplin - what\'s new', + description: 'News about the Joplin open source application', + url: 'https://joplinapp.org/news/', + }, }); } diff --git a/packages/tools/website/utils/openGraph.test.ts b/packages/tools/website/utils/openGraph.test.ts new file mode 100644 index 000000000..6128b6970 --- /dev/null +++ b/packages/tools/website/utils/openGraph.test.ts @@ -0,0 +1,46 @@ +import { extractOpenGraphTags, OpenGraphTags } from './openGraph'; +import { tempFilePath } from '@joplin/lib/testing/test-utils'; +import { writeFile } from 'fs-extra'; + +describe('openGraph', function() { + + it('should extract the Open Graph tags', async function() { + const tests: [string, OpenGraphTags][] = [ + + ['# My title\n\nMy note description **with bold text**', { + title: 'My title', + description: 'My note description with bold text', + url: 'https://test.com', + }], + + ['# My very very very very very very very very very very very very long title\n\nMy description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters.', { + title: 'My very very very very very very very very very very very very long...', + description: 'My description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters. My description that i...', + url: 'https://test.com', + }], + + ['# Page with an image\n\nSome text followed by an image. ![](https://test.com/hello1.png) ![](https://test.com/hello2.png)', { + title: 'Page with an image', + description: 'Some text followed by an image.', + url: 'https://test.com', + image: 'https://test.com/hello1.png', + }], + + ['# Image without domain\n\n![](/hello1.png)', { + title: 'Image without domain', + description: '', + url: 'https://test.com', + image: 'https://joplinapp.org/hello1.png', + }], + ]; + + for (const test of tests) { + const [input, expected] = test; + const filePath = await tempFilePath('md'); + await writeFile(filePath, input); + const actual = await extractOpenGraphTags(filePath, 'https://test.com'); + expect(actual).toEqual(expected); + } + }); + +}); diff --git a/packages/tools/website/utils/openGraph.ts b/packages/tools/website/utils/openGraph.ts new file mode 100644 index 000000000..54f608a15 --- /dev/null +++ b/packages/tools/website/utils/openGraph.ts @@ -0,0 +1,45 @@ +import { MarkupLanguage, MarkupToHtml } from '@joplin/renderer'; +import { readmeFileTitleAndBody, replaceGitHubByWebsiteLinks } from './parser'; +import markupLanguageUtils from '@joplin/lib/markupLanguageUtils'; +const { substrWithEllipsis } = require('@joplin/lib/string-utils'); + +export interface OpenGraphTags { + title: string; + description: string; + url: string; + image?: string; +} + +let markupToHtml_: MarkupToHtml = null; + +const markupToHtml = (): MarkupToHtml => { + if (!markupToHtml_) markupToHtml_ = new MarkupToHtml(); + return markupToHtml_; +}; + +const getImageUrl = (content: string): string | null => { + const imageUrls = markupLanguageUtils.extractImageUrls(MarkupLanguage.Any, content); + if (!imageUrls.length) return null; + let imageUrl = replaceGitHubByWebsiteLinks(imageUrls[0]); + if (!imageUrl.startsWith('https:')) { + if (!imageUrl.startsWith('/')) imageUrl = `/${imageUrl}`; + imageUrl = `https://joplinapp.org${imageUrl}`; + } + return imageUrl; +}; + +export const extractOpenGraphTags = async (sourcePath: string, url: string): Promise => { + const doc = await readmeFileTitleAndBody(sourcePath); + + const output: OpenGraphTags = { + title: substrWithEllipsis(doc.title, 0, 70), + description: substrWithEllipsis(markupToHtml().stripMarkup(MarkupLanguage.Markdown, doc.body), 0, 200), + url, + }; + + const imageUrl = getImageUrl(doc.body); + + if (imageUrl) output.image = imageUrl; + + return output; +}; diff --git a/packages/tools/website/utils/parser.ts b/packages/tools/website/utils/parser.ts new file mode 100644 index 000000000..6d1631c8f --- /dev/null +++ b/packages/tools/website/utils/parser.ts @@ -0,0 +1,37 @@ +import { readFile } from 'fs-extra'; +import { stripOffFrontMatter } from './frontMatter'; + +interface ReadmeDoc { + title: string; + body: string; +} + +export async function readmeFileTitleAndBody(sourcePath: string): Promise { + let md = await readFile(sourcePath, 'utf8'); + md = stripOffFrontMatter(md).doc.trim(); + const r = md.match(/^# (.*)/); + + if (!r) { + throw new Error(`Could not determine title for Markdown file: ${sourcePath}`); + } else { + const lines = md.split('\n'); + lines.splice(0, 1); + return { + title: r[1], + body: lines.join('\n'), + }; + } +} + +export async function readmeFileTitle(sourcePath: string) { + const r = await readmeFileTitleAndBody(sourcePath); + return r.title; +} + +export function replaceGitHubByWebsiteLinks(md: string) { + return md + .replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\/index\.md(#[^\s)]+|)/g, '/$1/$2') + .replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\.md(#[^\s)]+|)/g, '/$1/$2') + .replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/README\.md(#[^\s)]+|)/g, '/help/$1') + .replace(/https:\/\/raw.githubusercontent.com\/laurent22\/joplin\/dev\/Assets\/WebsiteAssets\/(.*?)/g, '/$1'); +} diff --git a/packages/tools/website/utils/types.ts b/packages/tools/website/utils/types.ts index 829e72ca4..154cea022 100644 --- a/packages/tools/website/utils/types.ts +++ b/packages/tools/website/utils/types.ts @@ -1,4 +1,5 @@ import { Plan, StripePublicConfig } from '@joplin/lib/utils/joplinCloud'; +import { OpenGraphTags } from './openGraph'; export enum Env { Dev = 'dev', @@ -72,6 +73,7 @@ export interface TemplateParams { assetUrls: AssetUrls; discussOnForumLink?: string; showBottomLinks?: boolean; + openGraph: OpenGraphTags; } export interface PlanPageParams extends TemplateParams {