All: Refactor Markdown rendering (#1315)

* Refactoring MdToHtml to avoid manually rendering tokens

* Minor fix

* Fixed loading of resources

* Handle clicking on checkboxes

* Added back Katex support

* Fixed issues with Katex and note rendering

* Added back support for links

* Restored code block highlighting support

* clean up

* Applying update to mobile

* Fixed handling of links and cleaned up to share more code between mobile and desktop

* Restored content caching and improved handling of additional assets

* Clean up and a few fixes

* Applied more updates to mobile and added code highlighting support

ReactNativeClient/lib/MdToHtml/rules/checkbox.js

@@ -0,0 +1,105 @@
+const Entities = require('html-entities').AllHtmlEntities;
+const htmlentities = (new Entities()).encode;
+const Resource = require('lib/models/Resource.js');
+const utils = require('../utils');
+
+let checkboxIndex_ = -1;
+
+function createPrefixTokens(Token, id, checked, label, postMessageSyntax, sourceToken) {
+	let token = null;
+	const tokens = [];
+
+	// A bit hard to handle errors here and it's unlikely that the token won't have a valid
+	// map parameter, but if it does set it to a very high value, which will be more easy to notice
+	// in calling code.
+	const lineIndex = sourceToken.map && sourceToken.map.length ? sourceToken.map[0] : 99999999;
+	const checkedString = checked ? 'checked' : 'unchecked';
+	const js = postMessageSyntax + "('checkboxclick:" + checkedString + ':' + lineIndex + "'); return true;";
+
+	token = new Token('checkbox_input', 'input', 0);
+	token.attrs = [
+		['type', 'checkbox'],
+		['id', id],
+		['onclick', js],
+	];
+	if (checked) token.attrs.push(['checked', 'true']);
+	tokens.push(token);
+
+	token = new Token('label_open', 'label', 1);
+	token.attrs = [['for', id]];
+	tokens.push(token);
+
+	if (label) {
+		token = new Token('text', '', 0);
+		token.content = label;
+		tokens.push(token);
+	}
+
+	return tokens;
+}
+
+function createSuffixTokens(Token) {
+	return [new Token('label_close', 'label', -1)];
+}
+
+function installRule(markdownIt, mdOptions, ruleOptions) {
+	markdownIt.core.ruler.push('checkbox', state => {
+		const tokens = state.tokens;
+		const Token = state.Token;
+
+		const checkboxPattern = /^\[([x|X| ])\] (.*)$/
+		let currentListItem = null;
+		let processedFirstInline = false;
+		for (let i = 0; i < tokens.length; i++) {
+			const token = tokens[i];
+
+			if (token.type === 'list_item_open') {
+				currentListItem = token;
+				processedFirstInline = false;
+				continue;
+			}
+
+			if (token.type === 'list_item_close') {
+				currentListItem = null;
+				processedFirstInline = false;
+				continue;
+			}
+
+			if (currentListItem && !processedFirstInline && token.type === 'inline') {
+				processedFirstInline = true;
+				const firstChild = token.children && token.children.length ? token.children[0] : null;
+				if (!firstChild) continue;
+
+				const matches = checkboxPattern.exec(firstChild.content);
+				if (!matches || matches.length < 2) continue;
+
+				checkboxIndex_++;
+				const checked = matches[1] !== ' ';
+				const id = 'md-checkbox-' + checkboxIndex_;
+				const label = matches.length >= 3 ? matches[2] : '';
+
+				// Prepend the text content with the checkbox markup and the opening <label> tag
+				// then append the </label> tag at the end of the text content.
+
+				const prefix = createPrefixTokens(Token, id, checked, label, ruleOptions.postMessageSyntax, token);
+				const suffix = createSuffixTokens(Token);
+
+				token.children = markdownIt.utils.arrayReplaceAt(token.children, 0, prefix);
+				token.children = token.children.concat(suffix);
+
+				// Add a class to the <li> container so that it can be targetted with CSS.
+
+				let itemClass = currentListItem.attrGet('class');
+				if (!itemClass) itemClass = '';
+				itemClass += ' md-checkbox';
+				currentListItem.attrSet('class', itemClass.trim());
+			}
+		}
+	});
+}
+
+module.exports = function(context, ruleOptions) {
+	return function(md, mdOptions) {
+		installRule(md, mdOptions, ruleOptions);
+	};
+};

ReactNativeClient/lib/MdToHtml/rules/highlight_keywords.js

@@ -0,0 +1,71 @@
+const Entities = require('html-entities').AllHtmlEntities;
+const htmlentities = (new Entities()).encode;
+const Resource = require('lib/models/Resource.js');
+const utils = require('../utils');
+const StringUtils = require('lib/string-utils.js');
+const md5 = require('md5');
+
+function createHighlightedTokens(Token, splitted) {
+	let token;
+	const output = [];
+
+	for (let i = 0; i < splitted.length; i++) {
+		const text = splitted[i];
+		if (!text) continue;
+
+		if (i % 2 === 0) {
+			token = new Token('text', '', 0);
+			token.content = text;
+			output.push(token);
+		} else {
+			token = new Token('highlighted_keyword_open', 'span', 1);
+			token.attrs = [['class', 'highlighted-keyword']];
+			output.push(token);
+
+			token = new Token('text', '', 0);
+			token.content = text;
+			output.push(token);
+
+			token = new Token('highlighted_keyword_close', 'span', -1);
+			output.push(token);
+		}
+	}
+
+	return output;
+}
+
+function installRule(markdownIt, mdOptions, ruleOptions) {
+	const divider = md5(Date.now().toString() + Math.random().toString());
+
+	markdownIt.core.ruler.push('highlight_keywords', state => {
+		const keywords = ruleOptions.highlightedKeywords;
+		if (!keywords || !keywords.length) return;
+
+		const tokens = state.tokens;
+		const Token = state.Token;
+
+		for (let i = 0; i < tokens.length; i++) {
+			const token = tokens[i];
+
+			if (token.type !== 'inline') continue;
+
+			for (let j = 0; j < token.children.length; j++) {
+				const child = token.children[j];
+				if (child.type !== 'text') continue;
+
+				const splitted = StringUtils.surroundKeywords(keywords, child.content, divider, divider).split(divider);
+				const splittedTokens = createHighlightedTokens(Token, splitted);
+				if (splittedTokens.length <= 1) continue;
+				
+				token.children = markdownIt.utils.arrayReplaceAt(token.children, j, splittedTokens);
+				j += splittedTokens.length - 1;
+			}
+		}
+	});
+}
+
+module.exports = function(context, ruleOptions) {
+	return function(md, mdOptions) {
+		installRule(md, mdOptions, ruleOptions);
+	};
+};

ReactNativeClient/lib/MdToHtml/rules/html_block.js

@@ -0,0 +1,45 @@
+const Entities = require('html-entities').AllHtmlEntities;
+const htmlentities = (new Entities()).encode;
+const Resource = require('lib/models/Resource.js');
+const utils = require('../utils');
+
+function renderImageHtml(before, src, after, ruleOptions) {
+	const resourceId = Resource.urlToId(src);
+	const resource = ruleOptions.resources[resourceId];
+	if (!resource) return '<div>' + utils.loaderImage() + '</div>';
+
+	const mime = resource.mime ? resource.mime.toLowerCase() : '';
+	if (Resource.isSupportedImageMimeType(mime)) {
+		let newSrc = './' + Resource.filename(resource);
+		if (ruleOptions.resourceBaseUrl) newSrc = ruleOptions.resourceBaseUrl + newSrc;
+		return '<img ' + before + ' data-resource-id="' + resource.id + '" src="' + newSrc + '" ' + after + '/>';
+	}
+
+	return '[Image: ' + htmlentities(resource.title) + ' (' + htmlentities(mime) + ')]';
+}
+
+function installRule(markdownIt, mdOptions, ruleOptions) {
+	const defaultRender = markdownIt.renderer.rules.html_block || function(tokens, idx, options, env, self) {
+		return self.renderToken(tokens, idx, options);
+	};
+
+	const imageRegex = /<img(.*?)src=["'](.*?)["'](.*?)\/>/
+
+	markdownIt.renderer.rules.html_block = function(tokens, idx, options, env, self) {
+		const token = tokens[idx];
+		const content = token.content;
+
+		if (!content.match(imageRegex)) return defaultRender(tokens, idx, options, env, self);
+		
+		return content.replace(imageRegex, (v, before, src, after) => {
+			if (!Resource.isResourceUrl(src)) return defaultRender(tokens, idx, options, env, self);
+			return renderImageHtml(before, src, after, ruleOptions);
+		});
+	};
+}
+
+module.exports = function(context, ruleOptions) {
+	return function(md, mdOptions) {
+		installRule(md, mdOptions, ruleOptions);
+	};
+};

ReactNativeClient/lib/MdToHtml/rules/html_inline.js

@@ -0,0 +1,39 @@
+// This rule is no longer needed because HTML anchors (as opposed to those generated from Markdown)
+// are handled in webviewLib. Keeping it here for reference.
+
+const Entities = require('html-entities').AllHtmlEntities;
+const htmlentities = (new Entities()).encode;
+const Resource = require('lib/models/Resource.js');
+const utils = require('../utils');
+
+function installRule(markdownIt, mdOptions, ruleOptions) {
+	const defaultRender = markdownIt.renderer.rules.html_block || function(tokens, idx, options, env, self) {
+		return self.renderToken(tokens, idx, options);
+	};
+
+	const anchorRegex = /<a (.*)>/
+
+	markdownIt.renderer.rules.html_inline = function(tokens, idx, options, env, self) {
+		const token = tokens[idx];
+		const content = token.content;
+
+		if (!content.match(anchorRegex)) return defaultRender(tokens, idx, options, env, self);
+
+		return content.replace(anchorRegex, (v, content) => {
+			let js = `
+				var href = this.getAttribute('href');
+				if (!href || href.indexOf('http') < 0) return true;
+				` + ruleOptions.postMessageSyntax + `(href);
+				return false;
+			`;
+			js = js.split('\n').join(' ').replace(/\t/g, '');
+			return '<a onclick="' + js + '" ' + content + '>';
+		});
+	};
+}
+
+module.exports = function(context, ruleOptions) {
+	return function(md, mdOptions) {
+		installRule(md, mdOptions, ruleOptions);
+	};
+};

ReactNativeClient/lib/MdToHtml/rules/image.js

@@ -0,0 +1,36 @@
+const Entities = require('html-entities').AllHtmlEntities;
+const htmlentities = (new Entities()).encode;
+const Resource = require('lib/models/Resource.js');
+const utils = require('../utils');
+
+function installRule(markdownIt, mdOptions, ruleOptions) {
+	const defaultRender = markdownIt.renderer.rules.image;
+
+	markdownIt.renderer.rules.image = (tokens, idx, options, env, self) => {
+		const token = tokens[idx];
+		const src = utils.getAttr(token.attrs, 'src');
+		const title = utils.getAttr(token.attrs, 'title');
+
+		if (!Resource.isResourceUrl(src)) return defaultRender(tokens, idx, options, env, self);
+
+		const resourceId = Resource.urlToId(src);
+		const resource = ruleOptions.resources[resourceId];
+		if (!resource) return '<div>' + utils.loaderImage() + '</div>';
+
+		const mime = resource.mime ? resource.mime.toLowerCase() : '';
+		if (Resource.isSupportedImageMimeType(mime)) {
+			let realSrc = './' + Resource.filename(resource);
+			if (ruleOptions.resourceBaseUrl) realSrc = ruleOptions.resourceBaseUrl + realSrc;
+			let output = '<img data-from-md data-resource-id="' + resource.id + '" title="' + htmlentities(title) + '" src="' + realSrc + '"/>';
+			return output;
+		}
+
+		return defaultRender(tokens, idx, options, env, self);
+	};
+}
+
+module.exports = function(context, ruleOptions) {
+	return function(md, mdOptions) {
+		installRule(md, mdOptions, ruleOptions);
+	};
+};

ReactNativeClient/lib/MdToHtml/rules/katex.js

@@ -0,0 +1,244 @@
+// Based on https://github.com/waylonflinn/markdown-it-katex
+
+'use strict';
+
+const { shim } = require('lib/shim');
+const Setting = require('lib/models/Setting');
+var katex = require('katex');
+const katexCss = require('lib/csstojs/katex.css.js');
+const md5 = require('md5');
+
+// Test if potential opening or closing delimieter
+// Assumes that there is a "$" at state.src[pos]
+function isValidDelim(state, pos) {
+	var prevChar, nextChar,
+		max = state.posMax,
+		can_open = true,
+		can_close = true;
+
+	prevChar = pos > 0 ? state.src.charCodeAt(pos - 1) : -1;
+	nextChar = pos + 1 <= max ? state.src.charCodeAt(pos + 1) : -1;
+
+	// Check non-whitespace conditions for opening and closing, and
+	// check that closing delimeter isn't followed by a number
+	if (prevChar === 0x20/* " " */ || prevChar === 0x09/* \t */ ||
+			(nextChar >= 0x30/* "0" */ && nextChar <= 0x39/* "9" */)) {
+		can_close = false;
+	}
+	if (nextChar === 0x20/* " " */ || nextChar === 0x09/* \t */) {
+		can_open = false;
+	}
+
+	return {
+		can_open: can_open,
+		can_close: can_close
+	};
+}
+
+function math_inline(state, silent) {
+	var start, match, token, res, pos, esc_count;
+
+	if (state.src[state.pos] !== "$") { return false; }
+
+	res = isValidDelim(state, state.pos);
+	if (!res.can_open) {
+		if (!silent) { state.pending += "$"; }
+		state.pos += 1;
+		return true;
+	}
+
+	// First check for and bypass all properly escaped delimieters
+	// This loop will assume that the first leading backtick can not
+	// be the first character in state.src, which is known since
+	// we have found an opening delimieter already.
+	start = state.pos + 1;
+	match = start;
+	while ( (match = state.src.indexOf("$", match)) !== -1) {
+		// Found potential $, look for escapes, pos will point to
+		// first non escape when complete
+		pos = match - 1;
+		while (state.src[pos] === "\\") { pos -= 1; }
+
+		// Even number of escapes, potential closing delimiter found
+		if ( ((match - pos) % 2) == 1 ) { break; }
+		match += 1;
+	}
+
+	// No closing delimter found.  Consume $ and continue.
+	if (match === -1) {
+		if (!silent) { state.pending += "$"; }
+		state.pos = start;
+		return true;
+	}
+
+	// Check if we have empty content, ie: $$.  Do not parse.
+	if (match - start === 0) {
+		if (!silent) { state.pending += "$$"; }
+		state.pos = start + 1;
+		return true;
+	}
+
+	// Check for valid closing delimiter
+	res = isValidDelim(state, match);
+	if (!res.can_close) {
+		if (!silent) { state.pending += "$"; }
+		state.pos = start;
+		return true;
+	}
+
+	if (!silent) {
+		token         = state.push('math_inline', 'math', 0);
+		token.markup  = "$";
+		token.content = state.src.slice(start, match);
+	}
+
+	state.pos = match + 1;
+	return true;
+}
+
+function math_block(state, start, end, silent){
+	var firstLine, lastLine, next, lastPos, found = false, token,
+		pos = state.bMarks[start] + state.tShift[start],
+		max = state.eMarks[start]
+
+	if(pos + 2 > max){ return false; }
+	if(state.src.slice(pos,pos+2)!=='$$'){ return false; }
+
+	pos += 2;
+	firstLine = state.src.slice(pos,max);
+
+	if(silent){ return true; }
+	if(firstLine.trim().slice(-2)==='$$'){
+		// Single line expression
+		firstLine = firstLine.trim().slice(0, -2);
+		found = true;
+	}
+
+	for(next = start; !found; ){
+
+		next++;
+
+		if(next >= end){ break; }
+
+		pos = state.bMarks[next]+state.tShift[next];
+		max = state.eMarks[next];
+
+		if(pos < max && state.tShift[next] < state.blkIndent){
+			// non-empty line with negative indent should stop the list:
+			break;
+		}
+
+		if(state.src.slice(pos,max).trim().slice(-2)==='$$'){
+			lastPos = state.src.slice(0,max).lastIndexOf('$$');
+			lastLine = state.src.slice(pos,lastPos);
+			found = true;
+		}
+
+	}
+
+	state.line = next + 1;
+
+	token = state.push('math_block', 'math', 0);
+	token.block = true;
+	token.content = (firstLine && firstLine.trim() ? firstLine + '\n' : '')
+	+ state.getLines(start + 1, next, state.tShift[start], true)
+	+ (lastLine && lastLine.trim() ? lastLine : '');
+	token.map = [ start, state.line ];
+	token.markup = '$$';
+	return true;
+}
+
+let assetsLoaded_ = false;
+let cache_ = {};
+
+module.exports = function(context, ruleOptions) {
+	// Keep macros that persist across Katex blocks to allow defining a macro
+	// in one block and re-using it later in other blocks.
+	// https://github.com/laurent22/joplin/issues/1105
+	context.__katex = { macros: {} };
+
+	const addContextAssets = () => {
+		context.css['katex'] = katexCss;
+		context.assetLoaders['katex'] = async () => {
+			if (assetsLoaded_) return;
+
+			// In node, the fonts are simply copied using copycss to where Katex expects to find them, which is under app/gui/note-viewer/fonts
+
+			// In React Native, it's more complicated and we need to download and copy them to the right directory. Ideally, we should embed
+			// them as an asset and copy them from there (or load them from there by modifying Katex CSS), but for now that will do.
+
+			if (shim.isReactNative()) {
+				// Fonts must go under the resourceDir directory because this is the baseUrl of NoteBodyViewer
+				const baseDir = Setting.value('resourceDir');
+				await shim.fsDriver().mkdir(baseDir + '/fonts');
+				
+				await shim.fetchBlob('https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.9.0-beta1/fonts/KaTeX_Main-Regular.woff2', { overwrite: false, path: baseDir + '/fonts/KaTeX_Main-Regular.woff2' });
+				await shim.fetchBlob('https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.9.0-beta1/fonts/KaTeX_Math-Italic.woff2', { overwrite: false, path: baseDir + '/fonts/KaTeX_Math-Italic.woff2' });
+				await shim.fetchBlob('https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.9.0-beta1/fonts/KaTeX_Size1-Regular.woff2', { overwrite: false, path: baseDir + '/fonts/KaTeX_Size1-Regular.woff2' });
+			}
+
+			assetsLoaded_ = true;
+		};
+	}
+
+	function renderToStringWithCache(latex, options) {
+		const cacheKey = md5(escape(latex) + escape(JSON.stringify(options)));
+		if (cacheKey in cache_) {
+			return cache_[cacheKey];
+		} else {
+			const beforeMacros = JSON.stringify(options.macros);
+			const output = katex.renderToString(latex, options);
+			const afterMacros = JSON.stringify(options.macros);
+
+			// Don't cache the formulas that add macros, otherwise
+			// they won't be added on second run.
+			if (beforeMacros === afterMacros) cache_[cacheKey] = output;
+			return output;
+		}
+	}
+
+	return function(md, options) {
+		// Default options
+
+		options = options || {};
+		options.macros = context.__katex.macros;
+
+		// set KaTeX as the renderer for markdown-it-simplemath
+		var katexInline = function(latex){
+			options.displayMode = false;
+			try{
+				return renderToStringWithCache(latex, options);
+			} catch(error){
+				if(options.throwOnError){ console.log(error); }
+				return latex;
+			}
+		};
+
+		var inlineRenderer = function(tokens, idx){
+			addContextAssets();
+			return katexInline(tokens[idx].content);
+		};
+
+		var katexBlock = function(latex){
+			options.displayMode = true;
+			try{
+				return "<p>" + renderToStringWithCache(latex, options) + "</p>";
+			} catch(error){
+				if(options.throwOnError){ console.log(error); }
+				return latex;
+			}
+		}
+
+		var blockRenderer = function(tokens, idx){
+			addContextAssets();
+			return katexBlock(tokens[idx].content) + '\n';
+		}
+
+		md.inline.ruler.after('escape', 'math_inline', math_inline);
+		md.block.ruler.after('blockquote', 'math_block', math_block, {
+			alt: [ 'paragraph', 'reference', 'blockquote', 'list' ]
+		});
+		md.renderer.rules.math_inline = inlineRenderer;
+		md.renderer.rules.math_block = blockRenderer;
+	};
+};

ReactNativeClient/lib/MdToHtml/rules/link_open.js

@@ -0,0 +1,45 @@
+const Entities = require('html-entities').AllHtmlEntities;
+const htmlentities = (new Entities()).encode;
+const Resource = require('lib/models/Resource.js');
+const utils = require('../utils');
+
+const loaderImage = '<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.0" width="16px" height="16px" viewBox="0 0 128 128" xml:space="preserve"><g><circle cx="16" cy="64" r="16" fill="#000000" fill-opacity="1"/><circle cx="16" cy="64" r="16" fill="#555555" fill-opacity="0.67" transform="rotate(45,64,64)"/><circle cx="16" cy="64" r="16" fill="#949494" fill-opacity="0.42" transform="rotate(90,64,64)"/><circle cx="16" cy="64" r="16" fill="#cccccc" fill-opacity="0.2" transform="rotate(135,64,64)"/><circle cx="16" cy="64" r="16" fill="#e1e1e1" fill-opacity="0.12" transform="rotate(180,64,64)"/><circle cx="16" cy="64" r="16" fill="#e1e1e1" fill-opacity="0.12" transform="rotate(225,64,64)"/><circle cx="16" cy="64" r="16" fill="#e1e1e1" fill-opacity="0.12" transform="rotate(270,64,64)"/><circle cx="16" cy="64" r="16" fill="#e1e1e1" fill-opacity="0.12" transform="rotate(315,64,64)"/><animateTransform attributeName="transform" type="rotate" values="0 64 64;315 64 64;270 64 64;225 64 64;180 64 64;135 64 64;90 64 64;45 64 64" calcMode="discrete" dur="720ms" repeatCount="indefinite"></animateTransform></g></svg>';
+
+function installRule(markdownIt, mdOptions, ruleOptions) {
+	markdownIt.renderer.rules.link_open = function (tokens, idx, options, env, self) {
+		const token = tokens[idx];
+		let href = utils.getAttr(token.attrs, 'href');
+		const text = utils.getAttr(token.attrs, 'text');
+		const isResourceUrl = Resource.isResourceUrl(href);
+		const title = isResourceUrl ? utils.getAttr(token.attrs, 'title') : href;
+
+		console.info(href, isResourceUrl);
+
+		let resourceIdAttr = "";
+		let icon = "";
+		let hrefAttr = '#';
+		if (isResourceUrl) {
+			const resourceId = Resource.pathToId(href);
+			href = "joplin://" + resourceId;
+			resourceIdAttr = "data-resource-id='" + resourceId + "'";
+			icon = '<span class="resource-icon"></span>';
+		} else {
+			// If the link is a plain URL (as opposed to a resource link), set the href to the actual
+			// link. This allows the link to be exported too when exporting to PDF. 
+			hrefAttr = href;
+		}
+
+		let js = ruleOptions.postMessageSyntax + "(" + JSON.stringify(href) + "); return false;";
+		if (hrefAttr.indexOf('#') === 0 && href.indexOf('#') === 0) js = ''; // If it's an internal anchor, don't add any JS since the webview is going to handle navigating to the right place
+		if (js) hrefAttr = '#';
+		let output = "<a data-from-md " + resourceIdAttr + " title='" + htmlentities(title) + "' href='" + hrefAttr + "' onclick='" + js + "'>" + icon;
+		console.info(output);
+		return output;
+	};
+}
+
+module.exports = function(context, ruleOptions) {
+	return function(md, mdOptions) {
+		installRule(md, mdOptions, ruleOptions);
+	};
+};