Desktop, Mobile: Add support for media player for video and audio files

Co-authored-by: Bryan <bryan.r.gerlach@gmail.com>

.eslintignore

@@ -145,6 +145,7 @@ ReactNativeClient/lib/hooks/useImperativeHandlerDebugger.js
 ReactNativeClient/lib/hooks/usePrevious.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/checkbox.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/fence.js
+ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/media.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/mermaid.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
 ReactNativeClient/lib/JoplinServerApi.js

.gitignore

@@ -136,6 +136,7 @@ ReactNativeClient/lib/hooks/useImperativeHandlerDebugger.js
 ReactNativeClient/lib/hooks/usePrevious.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/checkbox.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/fence.js
+ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/media.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/mermaid.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
 ReactNativeClient/lib/JoplinServerApi.js

ReactNativeClient/lib/joplin-renderer/HtmlToHtml.js

@@ -58,7 +58,7 @@ class HtmlToHtml {
 			html = htmlUtils.processImageTags(html, data => {
 				if (!data.src) return null;
 
-				const r = utils.imageReplacement(this.ResourceModel_, data.src, options.resources, this.resourceBaseUrl_);
+				const r = utils.resourceReplacement(this.ResourceModel_, data.src, options.resources, this.resourceBaseUrl_);
 				if (!r) return null;
 
 				if (typeof r === 'string') {

ReactNativeClient/lib/joplin-renderer/MdToHtml.js

@@ -17,6 +17,7 @@ const rules = {
 	code_inline: require('./MdToHtml/rules/code_inline'),
 	fountain: require('./MdToHtml/rules/fountain'),
 	mermaid: require('./MdToHtml/rules/mermaid').default,
+	media: require('./MdToHtml/rules/media').default,
 };
 
 const setupLinkify = require('./MdToHtml/setupLinkify');

ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/html_image.js

@@ -3,7 +3,7 @@ const htmlUtils = require('../../htmlUtils.js');
 const utils = require('../../utils');
 
 function renderImageHtml(before, src, after, ruleOptions) {
-	const r = utils.imageReplacement(ruleOptions.ResourceModel, src, ruleOptions.resources, ruleOptions.resourceBaseUrl);
+	const r = utils.resourceReplacement(ruleOptions.ResourceModel, src, ruleOptions.resources, ruleOptions.resourceBaseUrl);
 	if (typeof r === 'string') return r;
 	if (r) return `<img ${before} ${htmlUtils.attributesHtml(r)} ${after}/>`;
 	return `[Image: ${src}]`;

ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/image.js

@@ -14,9 +14,9 @@ function installRule(markdownIt, mdOptions, ruleOptions) {
 
 		if (!Resource.isResourceUrl(src) || ruleOptions.plainResourceRendering) return defaultRender(tokens, idx, options, env, self);
 
-		const r = utils.imageReplacement(ruleOptions.ResourceModel, src, ruleOptions.resources, ruleOptions.resourceBaseUrl);
+		const r = utils.resourceReplacement(ruleOptions.ResourceModel, src, ruleOptions.resources, ruleOptions.resourceBaseUrl);
 		if (typeof r === 'string') return r;
-		if (r) return `<img data-from-md ${htmlUtils.attributesHtml(Object.assign({}, r, { title: title }))}/>`;
+		if (r && r.type === 'image') return `<img data-from-md ${htmlUtils.attributesHtml(Object.assign({}, r, { title: title }))}/>`;
 
 		return defaultRender(tokens, idx, options, env, self);
 	};

ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/media.ts

@@ -0,0 +1,30 @@
+const utils = require('../../utils');
+
+// @ts-ignore: Keep the function signature as-is despite unusued arguments
+function installRule(markdownIt:any, mdOptions:any, ruleOptions:any, context:any) {
+	const defaultRender = markdownIt.renderer.rules.link_open;
+
+	markdownIt.renderer.rules.link_open = (tokens: { [x: string]: any; }, idx: string | number, options: any, env: any, self: any) => {
+		const Resource = ruleOptions.ResourceModel;
+
+		const token = tokens[idx];
+		const src = utils.getAttr(token.attrs, 'href');
+
+		if (!Resource.isResourceUrl(src) || ruleOptions.plainResourceRendering) return defaultRender(tokens, idx, options, env, self);
+
+		const r = utils.resourceReplacement(ruleOptions.ResourceModel, src, ruleOptions.resources, ruleOptions.resourceBaseUrl);
+		if (typeof r === 'string') return r;
+		if (r && r.type === 'audio') return `<audio controls><source src='${r.src}'></audio><a href=# onclick=ipcProxySendToHost('joplin://${src.substring(2)}')>`;
+		if (r && r.type === 'video') return `<video style="width:100%" controls><source src='${r.src}'></video>`;
+
+		console.log(context);
+
+		return defaultRender(tokens, idx, options, env, self);
+	};
+}
+
+export default function(context:any,ruleOptions:any) {
+	return function(md:any, mdOptions:any) {
+		installRule(md, mdOptions, ruleOptions, context);
+	};
+}

ReactNativeClient/lib/joplin-renderer/utils.js

@@ -122,7 +122,8 @@ utils.resourceStatus = function(ResourceModel, resourceInfo) {
 	return resourceStatus;
 };
 
-utils.imageReplacement = function(ResourceModel, src, resources, resourceBaseUrl) {
+utils.resourceReplacement = function(ResourceModel, src, resources, resourceBaseUrl) {
+	if (!ResourceModel) return null;
 	if (!ResourceModel || !resources) return null;
 
 	if (!ResourceModel.isResourceUrl(src)) return null;
@@ -138,13 +139,15 @@ utils.imageReplacement = function(ResourceModel, src, resources, resourceBaseUrl
 	}
 
 	const mime = resource.mime ? resource.mime.toLowerCase() : '';
-	if (ResourceModel.isSupportedImageMimeType(mime)) {
+	const type = ResourceModel.mimeTypeToMediaType(mime);
+	if (type != 'unknown') {
 		let newSrc = `./${ResourceModel.filename(resource)}`;
 		if (resourceBaseUrl) newSrc = resourceBaseUrl + newSrc;
 		newSrc += `?t=${resource.updated_time}`;
 		return {
 			'data-resource-id': resource.id,
 			src: newSrc,
+			type: type,
 		};
 	}
 

ReactNativeClient/lib/models/Resource.js

@@ -26,9 +26,16 @@ class Resource extends BaseItem {
 		return this.encryptionService_;
 	}
 
-	static isSupportedImageMimeType(type) {
-		const imageMimeTypes = ['image/jpg', 'image/jpeg', 'image/png', 'image/gif', 'image/svg+xml', 'image/webp'];
-		return imageMimeTypes.indexOf(type.toLowerCase()) >= 0;
+	static mimeTypeToMediaType(type) {
+		if (type.startsWith('image/')) {
+			return 'image';
+		} else if (type.startsWith('audio/')) {
+			return 'audio';
+		} else if (type.startsWith('video/')) {
+			return 'video';
+		} else {
+			return 'unknown';
+		}
 	}
 
 	static fetchStatuses(resourceIds) {
@@ -205,7 +212,7 @@ class Resource extends BaseItem {
 		let tagAlt = resource.alt ? resource.alt : resource.title;
 		if (!tagAlt) tagAlt = '';
 		const lines = [];
-		if (Resource.isSupportedImageMimeType(resource.mime)) {
+		if (Resource.mimeTypeToMediaType(resource.mime) === 'image') {
 			lines.push('![');
 			lines.push(markdownUtils.escapeTitleText(tagAlt));
 			lines.push(`](:/${resource.id})`);

ReactNativeClient/lib/models/Setting.js

@@ -477,6 +477,7 @@ class Setting extends BaseModel {
 			'markdown.plugin.emoji': { value: false, type: Setting.TYPE_BOOL, section: 'plugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable markdown emoji')}${wysiwygNo}` },
 			'markdown.plugin.insert': { value: false, type: Setting.TYPE_BOOL, section: 'plugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable ++insert++ syntax')}${wysiwygNo}` },
 			'markdown.plugin.multitable': { value: false, type: Setting.TYPE_BOOL, section: 'plugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable multimarkdown table extension')}${wysiwygNo}` },
+			'markdown.plugin.media': { value: true, type: Setting.TYPE_BOOL, section: 'plugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable media players (audio and video)')}${wysiwygNo}` },
 
 			// Tray icon (called AppIndicator) doesn't work in Ubuntu
 			// http://www.webupd8.org/2017/04/fix-appindicator-not-working-for.html