Tools: Add Open Graph tags to website

.eslintignore

@@ -2011,6 +2011,15 @@ packages/tools/website/updateDownloadPage.js.map
 packages/tools/website/utils/frontMatter.d.ts
 packages/tools/website/utils/frontMatter.js
 packages/tools/website/utils/frontMatter.js.map
+packages/tools/website/utils/openGraph.d.ts
+packages/tools/website/utils/openGraph.js
+packages/tools/website/utils/openGraph.js.map
+packages/tools/website/utils/openGraph.test.d.ts
+packages/tools/website/utils/openGraph.test.js
+packages/tools/website/utils/openGraph.test.js.map
+packages/tools/website/utils/parser.d.ts
+packages/tools/website/utils/parser.js
+packages/tools/website/utils/parser.js.map
 packages/tools/website/utils/pressCarousel.d.ts
 packages/tools/website/utils/pressCarousel.js
 packages/tools/website/utils/pressCarousel.js.map

.gitignore

@@ -2001,6 +2001,15 @@ packages/tools/website/updateDownloadPage.js.map
 packages/tools/website/utils/frontMatter.d.ts
 packages/tools/website/utils/frontMatter.js
 packages/tools/website/utils/frontMatter.js.map
+packages/tools/website/utils/openGraph.d.ts
+packages/tools/website/utils/openGraph.js
+packages/tools/website/utils/openGraph.js.map
+packages/tools/website/utils/openGraph.test.d.ts
+packages/tools/website/utils/openGraph.test.js
+packages/tools/website/utils/openGraph.test.js.map
+packages/tools/website/utils/parser.d.ts
+packages/tools/website/utils/parser.js
+packages/tools/website/utils/parser.js.map
 packages/tools/website/utils/pressCarousel.d.ts
 packages/tools/website/utils/pressCarousel.js
 packages/tools/website/utils/pressCarousel.js.map

Assets/WebsiteAssets/templates/front.mustache

@@ -10,8 +10,8 @@
 		<link rel="icon" href="{{imageBaseUrl}}/favicon.png" />
 		<meta name="viewport" content="width=device-width, initial-scale=1" />
 		<meta name="theme-color" content="#000000" />
-		<meta name="description" content="Joplin, the open source note-taking application" />
 		<link rel="stylesheet" href="{{{assetUrls.css.fontawesome}}}">
+		{{> openGraphTags}}
 		<link
 			rel="stylesheet"
 			href="{{cssBaseUrl}}/bootstrap5.0.2.min.css"

Assets/WebsiteAssets/templates/main-new.mustache

@@ -23,7 +23,7 @@ https://github.com/laurent22/joplin/blob/dev/{{{sourceMarkdownFile}}}
 		<link rel="icon" href="{{imageBaseUrl}}/favicon.png" />
 		<meta name="viewport" content="width=device-width, initial-scale=1" />
 		<meta name="theme-color" content="#000000" />
-		<meta name="description" content="Joplin, the open source note-taking application" />
+		{{> openGraphTags}}
 		<link
 			rel="stylesheet"
 			href="{{cssBaseUrl}}/bootstrap5.0.2.min.css"

Assets/WebsiteAssets/templates/partials/openGraphTags.mustache

@@ -0,0 +1,11 @@
+{{#openGraph}}
+	<meta name="description" content="{{openGraph.description}}" />
+	<meta name="twitter:card" content="summary" />
+	<meta name="twitter:site" content="@joplinapp" />
+	<meta property="og:url" content="{{{openGraph.url}}}" />
+	<meta property="og:title" content="{{openGraph.title}}" />
+	<meta property="og:description" content="{{openGraph.description}}" />
+	{{#openGraph.image}}
+		<meta property="og:image" content="{{{openGraph.image}}}" />
+	{{/openGraph.image}}
+{{/openGraph}}

packages/lib/markupLanguageUtils.ts

@@ -15,8 +15,17 @@ export class MarkupLanguageUtils {
 		throw new Error(`Unsupported markup language: ${language}`);
 	}
 
-	public extractImageUrls(language: MarkupLanguage, text: string) {
+	public extractImageUrls(language: MarkupLanguage, text: string): string[] {
-		return this.lib_(language).extractImageUrls(text);
+		let urls: string[] = [];
+
+		if (language === MarkupLanguage.Any) {
+			urls = urls.concat(this.lib_(MarkupLanguage.Markdown).extractImageUrls(text));
+			urls = urls.concat(this.lib_(MarkupLanguage.Html).extractImageUrls(text));
+		} else {
+			urls = this.lib_(language).extractImageUrls(text);
+		}
+
+		return urls;
 	}
 
 	// Create a new MarkupToHtml instance while injecting options specific to Joplin

packages/renderer/MarkupToHtml.ts

@@ -7,6 +7,7 @@ const MarkdownIt = require('markdown-it');
 export enum MarkupLanguage {
 	Markdown = 1,
 	Html = 2,
+	Any = 3,
 }
 
 export interface RenderResultPluginAsset {

packages/tools/website/build.ts

@@ -6,6 +6,8 @@ import { AssetUrls, Env, PlanPageParams, Sponsors, TemplateParams } from './util
 import { getPlans, loadStripeConfig } from '@joplin/lib/utils/joplinCloud';
 import { stripOffFrontMatter } from './utils/frontMatter';
 import { dirname, basename } from 'path';
+import { readmeFileTitle, replaceGitHubByWebsiteLinks } from './utils/parser';
+import { extractOpenGraphTags } from './utils/openGraph';
 const moment = require('moment');
 
 const glob = require('glob');
@@ -49,14 +51,6 @@ async function getDonateLinks() {
 	return `<div class="donate-links">\n\n${matches[1].trim()}\n\n</div>`;
 }
 
-function replaceGitHubByWebsiteLinks(md: string) {
-	return md
-		.replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\/index\.md(#[^\s)]+|)/g, '/$1/$2')
-		.replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\.md(#[^\s)]+|)/g, '/$1/$2')
-		.replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/README\.md(#[^\s)]+|)/g, '/help/$1')
-		.replace(/https:\/\/raw.githubusercontent.com\/laurent22\/joplin\/dev\/Assets\/WebsiteAssets\/(.*?)/g, '/$1');
-}
-
 function tocHtml() {
 	if (tocHtml_) return tocHtml_;
 	const markdownIt = getMarkdownIt();
@@ -104,6 +98,7 @@ function defaultTemplateParams(assetUrls: AssetUrls): TemplateParams {
 			isFrontPage: false,
 		},
 		assetUrls,
+		openGraph: null,
 	};
 }
 
@@ -142,18 +137,6 @@ function renderPageToHtml(md: string, targetPath: string, templateParams: Templa
 	writeFileSync(targetPath, html);
 }
 
-async function readmeFileTitle(sourcePath: string) {
-	let md = await readFile(sourcePath, 'utf8');
-	md = stripOffFrontMatter(md).doc;
-	const r = md.match(/(^|\n)# (.*)/);
-
-	if (!r) {
-		throw new Error(`Could not determine title for Markdown file: ${sourcePath}`);
-	} else {
-		return r[2];
-	}
-}
-
 function renderFileToHtml(sourcePath: string, targetPath: string, templateParams: TemplateParams) {
 	let md = readFileSync(sourcePath, 'utf8');
 	md = stripOffFrontMatter(md).doc;
@@ -229,6 +212,11 @@ async function main() {
 		partials,
 		sponsors,
 		assetUrls,
+		openGraph: {
+			title: 'Joplin documentation',
+			description: '',
+			url: 'https://joplinapp.org/help/',
+		},
 	});
 
 	// =============================================================
@@ -252,6 +240,11 @@ async function main() {
 		},
 		showToc: false,
 		assetUrls,
+		openGraph: {
+			title: 'Joplin website',
+			description: 'Joplin, the open source note-taking application',
+			url: 'https://joplinapp.org',
+		},
 	});
 
 	// =============================================================
@@ -290,17 +283,17 @@ async function main() {
 	const mdFiles = glob.sync(`${readmeDir}/**/*.md`).map((f: string) => f.substr(rootDir.length + 1));
 	const sources = [];
 
-	const makeTargetFilePath = (input: string): string => {
+	const makeTargetBasename = (input: string): string => {
 		if (isNewsFile(input)) {
 			const filenameNoExt = basename(input, '.md');
-			return `${docDir}/news/${filenameNoExt}/index.html`;
+			return `news/${filenameNoExt}/index.html`;
 		} else {
 			// Input is for example "readme/spec/interop_with_frontmatter.md",
 			// and we need to convert it to
 			// "docs/spec/interop_with_frontmatter/index.html" and prefix it
 			// with the website repo full path.
 
-			let s = `${docDir}/${input}`;
+			let s = input;
 			if (s.endsWith('index.md')) {
 				s = s.replace(/index\.md/, 'index.html');
 			} else {
@@ -313,11 +306,20 @@ async function main() {
 		}
 	};
 
+	const makeTargetFilePath = (input: string): string => {
+		return `${docDir}/${makeTargetBasename(input)}`;
+	};
+
+	const makeTargetUrl = (input: string) => {
+		return `https://joplinapp.org/${makeTargetBasename(input)}`;
+	};
+
 	const newsFilePaths: string[] = [];
 
 	for (const mdFile of mdFiles) {
 		const title = await readmeFileTitle(`${rootDir}/${mdFile}`);
 		const targetFilePath = makeTargetFilePath(mdFile);
+		const openGraph = await extractOpenGraphTags(mdFile, makeTargetUrl(mdFile));
 
 		const isNews = isNewsFile(mdFile);
 		if (isNews) newsFilePaths.push(mdFile);
@@ -326,6 +328,7 @@ async function main() {
 			title: title,
 			donateLinksMd: mdFile === 'readme/donate.md' ? '' : donateLinksMd,
 			showToc: mdFile !== 'readme/download.md' && !isNews,
+			openGraph,
 		}]);
 	}
 
@@ -354,11 +357,17 @@ async function main() {
 
 	await makeNewsFrontPage(newsFilePaths, `${docDir}/news/index.html`, {
 		...defaultTemplateParams(assetUrls),
+		title: 'What\'s new',
 		pageName: 'news',
 		partials,
 		showToc: false,
 		showImproveThisDoc: false,
 		donateLinksMd,
+		openGraph: {
+			title: 'Joplin - what\'s new',
+			description: 'News about the Joplin open source application',
+			url: 'https://joplinapp.org/news/',
+		},
 	});
 }
 

packages/tools/website/utils/openGraph.test.ts

@@ -0,0 +1,46 @@
+import { extractOpenGraphTags, OpenGraphTags } from './openGraph';
+import { tempFilePath } from '@joplin/lib/testing/test-utils';
+import { writeFile } from 'fs-extra';
+
+describe('openGraph', function() {
+
+	it('should extract the Open Graph tags', async function() {
+		const tests: [string, OpenGraphTags][] = [
+
+			['# My title\n\nMy note description **with bold text**', {
+				title: 'My title',
+				description: 'My note description with bold text',
+				url: 'https://test.com',
+			}],
+
+			['# My very very very very very very very very very very very very long title\n\nMy description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters.', {
+				title: 'My very very very very very very very very very very very very long...',
+				description: 'My description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters. My description that is over 200 characters. My description that i...',
+				url: 'https://test.com',
+			}],
+
+			['# Page with an image\n\nSome text followed by an image. ![](https://test.com/hello1.png) ![](https://test.com/hello2.png)', {
+				title: 'Page with an image',
+				description: 'Some text followed by an image.',
+				url: 'https://test.com',
+				image: 'https://test.com/hello1.png',
+			}],
+
+			['# Image without domain\n\n![](/hello1.png)', {
+				title: 'Image without domain',
+				description: '',
+				url: 'https://test.com',
+				image: 'https://joplinapp.org/hello1.png',
+			}],
+		];
+
+		for (const test of tests) {
+			const [input, expected] = test;
+			const filePath = await tempFilePath('md');
+			await writeFile(filePath, input);
+			const actual = await extractOpenGraphTags(filePath, 'https://test.com');
+			expect(actual).toEqual(expected);
+		}
+	});
+
+});

packages/tools/website/utils/openGraph.ts

@@ -0,0 +1,45 @@
+import { MarkupLanguage, MarkupToHtml } from '@joplin/renderer';
+import { readmeFileTitleAndBody, replaceGitHubByWebsiteLinks } from './parser';
+import markupLanguageUtils from '@joplin/lib/markupLanguageUtils';
+const { substrWithEllipsis } = require('@joplin/lib/string-utils');
+
+export interface OpenGraphTags {
+	title: string;
+	description: string;
+	url: string;
+	image?: string;
+}
+
+let markupToHtml_: MarkupToHtml = null;
+
+const markupToHtml = (): MarkupToHtml => {
+	if (!markupToHtml_) markupToHtml_ = new MarkupToHtml();
+	return markupToHtml_;
+};
+
+const getImageUrl = (content: string): string | null => {
+	const imageUrls = markupLanguageUtils.extractImageUrls(MarkupLanguage.Any, content);
+	if (!imageUrls.length) return null;
+	let imageUrl = replaceGitHubByWebsiteLinks(imageUrls[0]);
+	if (!imageUrl.startsWith('https:')) {
+		if (!imageUrl.startsWith('/')) imageUrl = `/${imageUrl}`;
+		imageUrl = `https://joplinapp.org${imageUrl}`;
+	}
+	return imageUrl;
+};
+
+export const extractOpenGraphTags = async (sourcePath: string, url: string): Promise<OpenGraphTags> => {
+	const doc = await readmeFileTitleAndBody(sourcePath);
+
+	const output: OpenGraphTags = {
+		title: substrWithEllipsis(doc.title, 0, 70),
+		description: substrWithEllipsis(markupToHtml().stripMarkup(MarkupLanguage.Markdown, doc.body), 0, 200),
+		url,
+	};
+
+	const imageUrl = getImageUrl(doc.body);
+
+	if (imageUrl) output.image = imageUrl;
+
+	return output;
+};

packages/tools/website/utils/parser.ts

@@ -0,0 +1,37 @@
+import { readFile } from 'fs-extra';
+import { stripOffFrontMatter } from './frontMatter';
+
+interface ReadmeDoc {
+	title: string;
+	body: string;
+}
+
+export async function readmeFileTitleAndBody(sourcePath: string): Promise<ReadmeDoc> {
+	let md = await readFile(sourcePath, 'utf8');
+	md = stripOffFrontMatter(md).doc.trim();
+	const r = md.match(/^# (.*)/);
+
+	if (!r) {
+		throw new Error(`Could not determine title for Markdown file: ${sourcePath}`);
+	} else {
+		const lines = md.split('\n');
+		lines.splice(0, 1);
+		return {
+			title: r[1],
+			body: lines.join('\n'),
+		};
+	}
+}
+
+export async function readmeFileTitle(sourcePath: string) {
+	const r = await readmeFileTitleAndBody(sourcePath);
+	return r.title;
+}
+
+export function replaceGitHubByWebsiteLinks(md: string) {
+	return md
+		.replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\/index\.md(#[^\s)]+|)/g, '/$1/$2')
+		.replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\.md(#[^\s)]+|)/g, '/$1/$2')
+		.replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/README\.md(#[^\s)]+|)/g, '/help/$1')
+		.replace(/https:\/\/raw.githubusercontent.com\/laurent22\/joplin\/dev\/Assets\/WebsiteAssets\/(.*?)/g, '/$1');
+}

packages/tools/website/utils/types.ts

@@ -1,4 +1,5 @@
 import { Plan, StripePublicConfig } from '@joplin/lib/utils/joplinCloud';
+import { OpenGraphTags } from './openGraph';
 
 export enum Env {
 	Dev = 'dev',
@@ -72,6 +73,7 @@ export interface TemplateParams {
 	assetUrls: AssetUrls;
 	discussOnForumLink?: string;
 	showBottomLinks?: boolean;
+	openGraph: OpenGraphTags;
 }
 
 export interface PlanPageParams extends TemplateParams {