Desktop: Add support for media players (video, audio and PDF)

.eslintignore

@@ -1330,6 +1330,9 @@ packages/renderer/MdToHtml/linkReplacement.js.map
 packages/renderer/MdToHtml/linkReplacement.test.d.ts
 packages/renderer/MdToHtml/linkReplacement.test.js
 packages/renderer/MdToHtml/linkReplacement.test.js.map
+packages/renderer/MdToHtml/renderMedia.d.ts
+packages/renderer/MdToHtml/renderMedia.js
+packages/renderer/MdToHtml/renderMedia.js.map
 packages/renderer/MdToHtml/rules/checkbox.d.ts
 packages/renderer/MdToHtml/rules/checkbox.js
 packages/renderer/MdToHtml/rules/checkbox.js.map
@@ -1354,6 +1357,9 @@ packages/renderer/MdToHtml/rules/image.js.map
 packages/renderer/MdToHtml/rules/katex.d.ts
 packages/renderer/MdToHtml/rules/katex.js
 packages/renderer/MdToHtml/rules/katex.js.map
+packages/renderer/MdToHtml/rules/link_close.d.ts
+packages/renderer/MdToHtml/rules/link_close.js
+packages/renderer/MdToHtml/rules/link_close.js.map
 packages/renderer/MdToHtml/rules/link_open.d.ts
 packages/renderer/MdToHtml/rules/link_open.js
 packages/renderer/MdToHtml/rules/link_open.js.map

.gitignore

@@ -1319,6 +1319,9 @@ packages/renderer/MdToHtml/linkReplacement.js.map
 packages/renderer/MdToHtml/linkReplacement.test.d.ts
 packages/renderer/MdToHtml/linkReplacement.test.js
 packages/renderer/MdToHtml/linkReplacement.test.js.map
+packages/renderer/MdToHtml/renderMedia.d.ts
+packages/renderer/MdToHtml/renderMedia.js
+packages/renderer/MdToHtml/renderMedia.js.map
 packages/renderer/MdToHtml/rules/checkbox.d.ts
 packages/renderer/MdToHtml/rules/checkbox.js
 packages/renderer/MdToHtml/rules/checkbox.js.map
@@ -1343,6 +1346,9 @@ packages/renderer/MdToHtml/rules/image.js.map
 packages/renderer/MdToHtml/rules/katex.d.ts
 packages/renderer/MdToHtml/rules/katex.js
 packages/renderer/MdToHtml/rules/katex.js.map
+packages/renderer/MdToHtml/rules/link_close.d.ts
+packages/renderer/MdToHtml/rules/link_close.js
+packages/renderer/MdToHtml/rules/link_close.js.map
 packages/renderer/MdToHtml/rules/link_open.d.ts
 packages/renderer/MdToHtml/rules/link_open.js
 packages/renderer/MdToHtml/rules/link_open.js.map

packages/app-mobile/android/app/src/main/java/net/cozic/joplin/MainApplication.java

@@ -14,6 +14,7 @@ import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 import net.cozic.joplin.share.SharePackage;
+import android.webkit.WebView;
 
 public class MainApplication extends Application implements ReactApplication {
 
@@ -68,6 +69,15 @@ public class MainApplication extends Application implements ReactApplication {
     
     SoLoader.init(this, /* native exopackage */ false);
     initializeFlipper(this, getReactNativeHost().getReactInstanceManager());
+
+    // To allow debugging the webview using the Chrome developer tools.
+    // Open chrome://inspect/#devices to view the device and connect to it
+    // IMPORTANT: USB debugging must be enabled on the device for it to work.
+    // https://github.com/react-native-webview/react-native-webview/blob/master/docs/Debugging.md
+
+    if (BuildConfig.DEBUG) {
+      WebView.setWebContentsDebuggingEnabled(true);
+    }
   }
 
   /**

packages/app-mobile/components/NoteBodyViewer/hooks/useSource.ts

@@ -15,14 +15,6 @@ interface UseSourceResult {
 	injectedJs: string[];
 }
 
-let markupToHtml_: any = null;
-
-function markupToHtml() {
-	if (markupToHtml_) return markupToHtml_;
-	markupToHtml_ = markupLanguageUtils.newMarkupToHtml();
-	return markupToHtml_;
-}
-
 function usePrevious(value: any, initialValue: any = null): any {
 	const ref = useRef(initialValue);
 	useEffect(() => {
@@ -45,6 +37,10 @@ export default function useSource(noteBody: string, noteMarkupLanguage: number,
 		};
 	}, [themeId, paddingBottom]);
 
+	const markupToHtml = useMemo(() => {
+		return markupLanguageUtils.newMarkupToHtml();
+	}, [isFirstRender]);
+
 	// To address https://github.com/laurent22/joplin/issues/433
 	//
 	// If a checkbox in a note is ticked, the body changes, which normally
@@ -58,7 +54,7 @@ export default function useSource(noteBody: string, noteMarkupLanguage: number,
 	//
 	// IMPORTANT: KEEP noteBody AS THE FIRST dependency in the array as the
 	// below logic rely on this.
-	const effectDependencies = [noteBody, resourceLoadedTime, noteMarkupLanguage, themeId, rendererTheme, highlightedKeywords, noteResources, noteHash, isFirstRender];
+	const effectDependencies = [noteBody, resourceLoadedTime, noteMarkupLanguage, themeId, rendererTheme, highlightedKeywords, noteResources, noteHash, isFirstRender, markupToHtml];
 	const previousDeps = usePrevious(effectDependencies, []);
 	const changedDeps = effectDependencies.reduce((accum: any, dependency: any, index: any) => {
 		if (dependency !== previousDeps[index]) {
@@ -94,9 +90,9 @@ export default function useSource(noteBody: string, noteMarkupLanguage: number,
 			// it doesn't contain info about the resource download state. Because of that, if we were to use the markupToHtml() cache
 			// it wouldn't re-render at all. We don't need this cache in any way because this hook is only triggered when we know
 			// something has changed.
-			markupToHtml().clearCache(noteMarkupLanguage);
+			markupToHtml.clearCache(noteMarkupLanguage);
 
-			const result = await markupToHtml().render(
+			const result = await markupToHtml.render(
 				noteMarkupLanguage,
 				bodyToRender,
 				rendererTheme,

packages/lib/models/Setting.ts

@@ -596,6 +596,9 @@ class Setting extends BaseModel {
 			'markdown.plugin.fountain': { value: false, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable Fountain syntax support')}${wysiwygYes}` },
 			'markdown.plugin.mermaid': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable Mermaid diagrams support')}${wysiwygYes}` },
 
+			'markdown.plugin.audioPlayer': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable audio player')}${wysiwygNo}` },
+			'markdown.plugin.videoPlayer': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable video player')}${wysiwygNo}` },
+			'markdown.plugin.pdfViewer': { value: !mobilePlatform, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['desktop'], label: () => `${_('Enable PDF viewer')}${wysiwygNo}` },
 			'markdown.plugin.mark': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable ==mark== syntax')}${wysiwygNo}` },
 			'markdown.plugin.footnote': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable footnotes')}${wysiwygNo}` },
 			'markdown.plugin.toc': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable table of contents extension')}${wysiwygNo}` },

packages/renderer/HtmlToHtml.ts

@@ -135,11 +135,11 @@ export default class HtmlToHtml {
 					enableLongPress: options.enableLongPress,
 				});
 
-				if (!r) return null;
+				if (!r.html) return null;
 
 				return {
 					type: 'replaceElement',
-					html: r,
+					html: r.html,
 				};
 			});
 		}

packages/renderer/MdToHtml.ts

@@ -32,6 +32,7 @@ const rules: RendererRules = {
 	checkbox: require('./MdToHtml/rules/checkbox').default,
 	katex: require('./MdToHtml/rules/katex').default,
 	link_open: require('./MdToHtml/rules/link_open').default,
+	link_close: require('./MdToHtml/rules/link_close').default,
 	html_image: require('./MdToHtml/rules/html_image').default,
 	highlight_keywords: require('./MdToHtml/rules/highlight_keywords').default,
 	code_inline: require('./MdToHtml/rules/code_inline').default,
@@ -96,11 +97,19 @@ interface PluginAssets {
 	[pluginName: string]: PluginAsset[];
 }
 
+export interface Link {
+	href: string;
+	resource: any;
+	resourceReady: boolean;
+	resourceFullPath: string;
+}
+
 interface PluginContext {
 	css: any;
 	pluginAssets: any;
 	cache: any;
 	userData: any;
+	currentLinks: Link[];
 }
 
 interface RenderResultPluginAsset {
@@ -142,6 +151,10 @@ export interface RuleOptions {
 	// linkRenderingType = 1 is the regular rendering and clicking on it is handled via embedded JS (in onclick attribute)
 	// linkRenderingType = 2 gives a plain link with no JS. Caller needs to handle clicking on the link.
 	linkRenderingType?: number;
+
+	audioPlayerEnabled: boolean;
+	videoPlayerEnabled: boolean;
+	pdfViewerEnabled: boolean;
 }
 
 export default class MdToHtml {
@@ -201,10 +214,16 @@ export default class MdToHtml {
 	}
 
 	private pluginOptions(name: string) {
+		// Currently link_close is only used to append the media player to
+		// the resource links so we use the mediaPlayers plugin options for
+		// it.
+		if (name === 'link_close') name = 'mediaPlayers';
+
 		let o = this.pluginOptions_[name] ? this.pluginOptions_[name] : {};
 		o = Object.assign({
 			enabled: true,
 		}, o);
+
 		return o;
 	}
 
@@ -348,6 +367,10 @@ export default class MdToHtml {
 			codeTheme: 'atom-one-light.css',
 			theme: Object.assign({}, defaultNoteStyle, theme),
 			plugins: {},
+
+			audioPlayerEnabled: this.pluginEnabled('audioPlayer'),
+			videoPlayerEnabled: this.pluginEnabled('videoPlayer'),
+			pdfViewerEnabled: this.pluginEnabled('pdfViewer'),
 		}, options);
 
 		// The "codeHighlightCacheKey" option indicates what set of cached object should be
@@ -373,6 +396,7 @@ export default class MdToHtml {
 			pluginAssets: {},
 			cache: this.contextCache_,
 			userData: {},
+			currentLinks: [],
 		};
 
 		const markdownIt = new MarkdownIt({

packages/renderer/MdToHtml/linkReplacement.test.ts

@@ -3,12 +3,12 @@ import linkReplacement from './linkReplacement';
 describe('linkReplacement', () => {
 
 	test('should handle non-resource links', () => {
-		const r = linkReplacement('https://example.com/test');
+		const r = linkReplacement('https://example.com/test').html;
 		expect(r).toBe('<a data-from-md href=\'https://example.com/test\' onclick=\'postMessage("https://example.com/test", { resourceId: "" }); return false;\'>');
 	});
 
 	test('should handle non-resource links - simple rendering', () => {
-		const r = linkReplacement('https://example.com/test', { linkRenderingType: 2 });
+		const r = linkReplacement('https://example.com/test', { linkRenderingType: 2 }).html;
 		expect(r).toBe('<a data-from-md href=\'https://example.com/test\'>');
 	});
 
@@ -25,7 +25,7 @@ describe('linkReplacement', () => {
 					},
 				},
 			},
-		});
+		}).html;
 
 		expect(r).toBe(`<a data-from-md data-resource-id='${resourceId}' href='#' onclick='postMessage("joplin://${resourceId}", { resourceId: "${resourceId}" }); return false;'><span class="resource-icon fa-joplin"></span>`);
 	});
@@ -43,7 +43,7 @@ describe('linkReplacement', () => {
 					},
 				},
 			},
-		});
+		}).html;
 
 		// Since the icon is embedded as SVG, we only check for the prefix
 		const expectedPrefix = `<a class="not-loaded-resource resource-status-notDownloaded" data-resource-id="${resourceId}"><img src="data:image/svg+xml;utf8`;

packages/renderer/MdToHtml/linkReplacement.ts

@@ -14,7 +14,14 @@ export interface Options {
 	enableLongPress?: boolean;
 }
 
-export default function(href: string, options: Options = null) {
+export interface LinkReplacementResult {
+	html: string;
+	resource: any;
+	resourceReady: boolean;
+	resourceFullPath: string;
+}
+
+export default function(href: string, options: Options = null): LinkReplacementResult {
 	options = {
 		title: '',
 		resources: {},
@@ -35,6 +42,7 @@ export default function(href: string, options: Options = null) {
 	let hrefAttr = '#';
 	let mime = '';
 	let resourceId = '';
+	let resource = null;
 	if (isResourceUrl) {
 		resourceId = resourceHrefInfo.itemId;
 
@@ -44,11 +52,18 @@ export default function(href: string, options: Options = null) {
 		if (result && result.item) {
 			if (!title) title = result.item.title;
 			mime = result.item.mime;
+			resource = result.item;
 		}
 
 		if (result && resourceStatus !== 'ready' && !options.plainResourceRendering) {
 			const icon = utils.resourceStatusFile(resourceStatus);
-			return `<a class="not-loaded-resource resource-status-${resourceStatus}" data-resource-id="${resourceId}">` + `<img src="data:image/svg+xml;utf8,${htmlentities(icon)}"/>`;
+
+			return {
+				resourceReady: false,
+				html: `<a class="not-loaded-resource resource-status-${resourceStatus}" data-resource-id="${resourceId}">` + `<img src="data:image/svg+xml;utf8,${htmlentities(icon)}"/>`,
+				resource,
+				resourceFullPath: null,
+			};
 		} else {
 			href = `joplin://${resourceId}`;
 			if (resourceHrefInfo.hash) href += `#${resourceHrefInfo.hash}`;
@@ -100,5 +115,10 @@ export default function(href: string, options: Options = null) {
 		if (js) attrHtml.push(js);
 	}
 
-	return `<a ${attrHtml.join(' ')}>${icon}`;
+	return {
+		html: `<a ${attrHtml.join(' ')}>${icon}`,
+		resourceReady: true,
+		resource,
+		resourceFullPath: resource && options?.ResourceModel?.fullPath ? options.ResourceModel.fullPath(resource) : null,
+	};
 }

packages/renderer/MdToHtml/renderMedia.ts

@@ -0,0 +1,41 @@
+import { Link } from '../MdToHtml';
+import { toForwardSlashes } from '../pathUtils';
+const Entities = require('html-entities').AllHtmlEntities;
+const htmlentities = new Entities().encode;
+
+export interface Options {
+	audioPlayerEnabled: boolean;
+	videoPlayerEnabled: boolean;
+	pdfViewerEnabled: boolean;
+}
+
+export default function(link: Link, options: Options) {
+	const resource = link.resource;
+
+	if (!link.resourceReady || !resource || !resource.mime) return '';
+
+	const escapedResourcePath = htmlentities(`file://${toForwardSlashes(link.resourceFullPath)}`);
+	const escapedMime = htmlentities(resource.mime);
+
+	if (options.videoPlayerEnabled && resource.mime.indexOf('video/') === 0) {
+		return `
+			<video class="media-player media-video" controls>
+				<source src="${escapedResourcePath}" type="${escapedMime}">
+			</video>
+		`;
+	}
+
+	if (options.audioPlayerEnabled && resource.mime.indexOf('audio/') === 0) {
+		return `
+			<audio class="media-player media-audio" controls>
+				<source src="${escapedResourcePath}" type="${escapedMime}">
+			</audio>
+		`;
+	}
+
+	if (options.pdfViewerEnabled && resource.mime === 'application/pdf') {
+		return `<object data="${escapedResourcePath}" class="media-player media-pdf" type="${escapedMime}"></object>`;
+	}
+
+	return '';
+}

packages/renderer/MdToHtml/rules/link_close.ts

@@ -0,0 +1,22 @@
+// This rule is used to add a media player for certain resource types below
+// the link.
+
+import { RuleOptions } from '../../MdToHtml';
+import renderMedia, { Options as RenderMediaOptions } from '../renderMedia';
+
+function plugin(markdownIt: any, ruleOptions: RuleOptions) {
+	const defaultRender = markdownIt.renderer.rules.link_close || function(tokens: any, idx: any, options: any, _env: any, self: any) {
+		return self.renderToken(tokens, idx, options);
+	};
+
+	markdownIt.renderer.rules.link_close = function(tokens: any[], idx: number, options: any, env: any, self: any) {
+		const defaultOutput = defaultRender(tokens, idx, options, env, self);
+		const link = ruleOptions.context.currentLinks.pop();
+
+		if (!link || ruleOptions.linkRenderingType === 2 || ruleOptions.plainResourceRendering) return defaultOutput;
+
+		return [defaultOutput, renderMedia(link, ruleOptions as RenderMediaOptions)].join('');
+	};
+}
+
+export default { plugin };

packages/renderer/MdToHtml/rules/link_open.ts

@@ -12,7 +12,7 @@ function plugin(markdownIt: any, ruleOptions: RuleOptions) {
 		const isResourceUrl = ruleOptions.resources && !!resourceHrefInfo;
 		const title = utils.getAttr(token.attrs, 'title', isResourceUrl ? '' : href);
 
-		return linkReplacement(href, {
+		const replacement = linkReplacement(href, {
 			title,
 			resources: ruleOptions.resources,
 			ResourceModel: ruleOptions.ResourceModel,
@@ -21,6 +21,15 @@ function plugin(markdownIt: any, ruleOptions: RuleOptions) {
 			postMessageSyntax: ruleOptions.postMessageSyntax,
 			enableLongPress: ruleOptions.enableLongPress,
 		});
+
+		ruleOptions.context.currentLinks.push({
+			href: href,
+			resource: replacement.resource,
+			resourceReady: replacement.resourceReady,
+			resourceFullPath: replacement.resourceFullPath,
+		});
+
+		return replacement.html;
 	};
 }
 

packages/renderer/noteStyle.ts

@@ -333,6 +333,15 @@ export default function(theme: any) {
 			pointer-events: none;
 		}
 
+		.media-player {
+			width: 100%;
+			margin-top: 10px;
+		}
+
+		.media-player.media-pdf {
+			min-height: 100vh;
+		}
+
 		/* Clear the CODE style if the element is within a joplin-editable block */
 		.mce-content-body .joplin-editable code {
 			border: none;

packages/renderer/pathUtils.ts

@@ -28,3 +28,7 @@ export function fileExtension(path: string) {
 	if (output.length <= 1) return '';
 	return output[output.length - 1];
 }
+
+export function toForwardSlashes(path: string) {
+	return path.replace(/\\/g, '/');
+}