From 13dbeb4b36ff69aaa2f6913f13f8524d4a379634 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Wed, 9 Dec 2020 21:30:51 +0000 Subject: [PATCH] Desktop: Add support for media players (video, audio and PDF) --- .eslintignore | 6 +++ .gitignore | 6 +++ .../net/cozic/joplin/MainApplication.java | 10 +++++ .../NoteBodyViewer/hooks/useSource.ts | 18 ++++---- packages/lib/models/Setting.ts | 3 ++ packages/renderer/HtmlToHtml.ts | 4 +- packages/renderer/MdToHtml.ts | 24 +++++++++++ .../renderer/MdToHtml/linkReplacement.test.ts | 8 ++-- packages/renderer/MdToHtml/linkReplacement.ts | 26 ++++++++++-- packages/renderer/MdToHtml/renderMedia.ts | 41 +++++++++++++++++++ .../renderer/MdToHtml/rules/link_close.ts | 22 ++++++++++ packages/renderer/MdToHtml/rules/link_open.ts | 11 ++++- packages/renderer/noteStyle.ts | 9 ++++ packages/renderer/pathUtils.ts | 4 ++ 14 files changed, 171 insertions(+), 21 deletions(-) create mode 100644 packages/renderer/MdToHtml/renderMedia.ts create mode 100644 packages/renderer/MdToHtml/rules/link_close.ts diff --git a/.eslintignore b/.eslintignore index bf77bff48..3c4b2044f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1330,6 +1330,9 @@ packages/renderer/MdToHtml/linkReplacement.js.map packages/renderer/MdToHtml/linkReplacement.test.d.ts packages/renderer/MdToHtml/linkReplacement.test.js packages/renderer/MdToHtml/linkReplacement.test.js.map +packages/renderer/MdToHtml/renderMedia.d.ts +packages/renderer/MdToHtml/renderMedia.js +packages/renderer/MdToHtml/renderMedia.js.map packages/renderer/MdToHtml/rules/checkbox.d.ts packages/renderer/MdToHtml/rules/checkbox.js packages/renderer/MdToHtml/rules/checkbox.js.map @@ -1354,6 +1357,9 @@ packages/renderer/MdToHtml/rules/image.js.map packages/renderer/MdToHtml/rules/katex.d.ts packages/renderer/MdToHtml/rules/katex.js packages/renderer/MdToHtml/rules/katex.js.map +packages/renderer/MdToHtml/rules/link_close.d.ts +packages/renderer/MdToHtml/rules/link_close.js +packages/renderer/MdToHtml/rules/link_close.js.map packages/renderer/MdToHtml/rules/link_open.d.ts packages/renderer/MdToHtml/rules/link_open.js packages/renderer/MdToHtml/rules/link_open.js.map diff --git a/.gitignore b/.gitignore index fe1134146..76b611a38 100644 --- a/.gitignore +++ b/.gitignore @@ -1319,6 +1319,9 @@ packages/renderer/MdToHtml/linkReplacement.js.map packages/renderer/MdToHtml/linkReplacement.test.d.ts packages/renderer/MdToHtml/linkReplacement.test.js packages/renderer/MdToHtml/linkReplacement.test.js.map +packages/renderer/MdToHtml/renderMedia.d.ts +packages/renderer/MdToHtml/renderMedia.js +packages/renderer/MdToHtml/renderMedia.js.map packages/renderer/MdToHtml/rules/checkbox.d.ts packages/renderer/MdToHtml/rules/checkbox.js packages/renderer/MdToHtml/rules/checkbox.js.map @@ -1343,6 +1346,9 @@ packages/renderer/MdToHtml/rules/image.js.map packages/renderer/MdToHtml/rules/katex.d.ts packages/renderer/MdToHtml/rules/katex.js packages/renderer/MdToHtml/rules/katex.js.map +packages/renderer/MdToHtml/rules/link_close.d.ts +packages/renderer/MdToHtml/rules/link_close.js +packages/renderer/MdToHtml/rules/link_close.js.map packages/renderer/MdToHtml/rules/link_open.d.ts packages/renderer/MdToHtml/rules/link_open.js packages/renderer/MdToHtml/rules/link_open.js.map diff --git a/packages/app-mobile/android/app/src/main/java/net/cozic/joplin/MainApplication.java b/packages/app-mobile/android/app/src/main/java/net/cozic/joplin/MainApplication.java index 1924a703c..dba576562 100644 --- a/packages/app-mobile/android/app/src/main/java/net/cozic/joplin/MainApplication.java +++ b/packages/app-mobile/android/app/src/main/java/net/cozic/joplin/MainApplication.java @@ -14,6 +14,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.util.List; import net.cozic.joplin.share.SharePackage; +import android.webkit.WebView; public class MainApplication extends Application implements ReactApplication { @@ -68,6 +69,15 @@ public class MainApplication extends Application implements ReactApplication { SoLoader.init(this, /* native exopackage */ false); initializeFlipper(this, getReactNativeHost().getReactInstanceManager()); + + // To allow debugging the webview using the Chrome developer tools. + // Open chrome://inspect/#devices to view the device and connect to it + // IMPORTANT: USB debugging must be enabled on the device for it to work. + // https://github.com/react-native-webview/react-native-webview/blob/master/docs/Debugging.md + + if (BuildConfig.DEBUG) { + WebView.setWebContentsDebuggingEnabled(true); + } } /** diff --git a/packages/app-mobile/components/NoteBodyViewer/hooks/useSource.ts b/packages/app-mobile/components/NoteBodyViewer/hooks/useSource.ts index 13a978cc8..092e1bb9b 100644 --- a/packages/app-mobile/components/NoteBodyViewer/hooks/useSource.ts +++ b/packages/app-mobile/components/NoteBodyViewer/hooks/useSource.ts @@ -15,14 +15,6 @@ interface UseSourceResult { injectedJs: string[]; } -let markupToHtml_: any = null; - -function markupToHtml() { - if (markupToHtml_) return markupToHtml_; - markupToHtml_ = markupLanguageUtils.newMarkupToHtml(); - return markupToHtml_; -} - function usePrevious(value: any, initialValue: any = null): any { const ref = useRef(initialValue); useEffect(() => { @@ -45,6 +37,10 @@ export default function useSource(noteBody: string, noteMarkupLanguage: number, }; }, [themeId, paddingBottom]); + const markupToHtml = useMemo(() => { + return markupLanguageUtils.newMarkupToHtml(); + }, [isFirstRender]); + // To address https://github.com/laurent22/joplin/issues/433 // // If a checkbox in a note is ticked, the body changes, which normally @@ -58,7 +54,7 @@ export default function useSource(noteBody: string, noteMarkupLanguage: number, // // IMPORTANT: KEEP noteBody AS THE FIRST dependency in the array as the // below logic rely on this. - const effectDependencies = [noteBody, resourceLoadedTime, noteMarkupLanguage, themeId, rendererTheme, highlightedKeywords, noteResources, noteHash, isFirstRender]; + const effectDependencies = [noteBody, resourceLoadedTime, noteMarkupLanguage, themeId, rendererTheme, highlightedKeywords, noteResources, noteHash, isFirstRender, markupToHtml]; const previousDeps = usePrevious(effectDependencies, []); const changedDeps = effectDependencies.reduce((accum: any, dependency: any, index: any) => { if (dependency !== previousDeps[index]) { @@ -94,9 +90,9 @@ export default function useSource(noteBody: string, noteMarkupLanguage: number, // it doesn't contain info about the resource download state. Because of that, if we were to use the markupToHtml() cache // it wouldn't re-render at all. We don't need this cache in any way because this hook is only triggered when we know // something has changed. - markupToHtml().clearCache(noteMarkupLanguage); + markupToHtml.clearCache(noteMarkupLanguage); - const result = await markupToHtml().render( + const result = await markupToHtml.render( noteMarkupLanguage, bodyToRender, rendererTheme, diff --git a/packages/lib/models/Setting.ts b/packages/lib/models/Setting.ts index 8b1990ede..30b562806 100644 --- a/packages/lib/models/Setting.ts +++ b/packages/lib/models/Setting.ts @@ -596,6 +596,9 @@ class Setting extends BaseModel { 'markdown.plugin.fountain': { value: false, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable Fountain syntax support')}${wysiwygYes}` }, 'markdown.plugin.mermaid': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable Mermaid diagrams support')}${wysiwygYes}` }, + 'markdown.plugin.audioPlayer': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable audio player')}${wysiwygNo}` }, + 'markdown.plugin.videoPlayer': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable video player')}${wysiwygNo}` }, + 'markdown.plugin.pdfViewer': { value: !mobilePlatform, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['desktop'], label: () => `${_('Enable PDF viewer')}${wysiwygNo}` }, 'markdown.plugin.mark': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable ==mark== syntax')}${wysiwygNo}` }, 'markdown.plugin.footnote': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable footnotes')}${wysiwygNo}` }, 'markdown.plugin.toc': { value: true, type: SettingItemType.Bool, section: 'markdownPlugins', public: true, appTypes: ['mobile', 'desktop'], label: () => `${_('Enable table of contents extension')}${wysiwygNo}` }, diff --git a/packages/renderer/HtmlToHtml.ts b/packages/renderer/HtmlToHtml.ts index ef8c0ee16..02b73cb98 100644 --- a/packages/renderer/HtmlToHtml.ts +++ b/packages/renderer/HtmlToHtml.ts @@ -135,11 +135,11 @@ export default class HtmlToHtml { enableLongPress: options.enableLongPress, }); - if (!r) return null; + if (!r.html) return null; return { type: 'replaceElement', - html: r, + html: r.html, }; }); } diff --git a/packages/renderer/MdToHtml.ts b/packages/renderer/MdToHtml.ts index 8fa2bc5c3..9955cad4e 100644 --- a/packages/renderer/MdToHtml.ts +++ b/packages/renderer/MdToHtml.ts @@ -32,6 +32,7 @@ const rules: RendererRules = { checkbox: require('./MdToHtml/rules/checkbox').default, katex: require('./MdToHtml/rules/katex').default, link_open: require('./MdToHtml/rules/link_open').default, + link_close: require('./MdToHtml/rules/link_close').default, html_image: require('./MdToHtml/rules/html_image').default, highlight_keywords: require('./MdToHtml/rules/highlight_keywords').default, code_inline: require('./MdToHtml/rules/code_inline').default, @@ -96,11 +97,19 @@ interface PluginAssets { [pluginName: string]: PluginAsset[]; } +export interface Link { + href: string; + resource: any; + resourceReady: boolean; + resourceFullPath: string; +} + interface PluginContext { css: any; pluginAssets: any; cache: any; userData: any; + currentLinks: Link[]; } interface RenderResultPluginAsset { @@ -142,6 +151,10 @@ export interface RuleOptions { // linkRenderingType = 1 is the regular rendering and clicking on it is handled via embedded JS (in onclick attribute) // linkRenderingType = 2 gives a plain link with no JS. Caller needs to handle clicking on the link. linkRenderingType?: number; + + audioPlayerEnabled: boolean; + videoPlayerEnabled: boolean; + pdfViewerEnabled: boolean; } export default class MdToHtml { @@ -201,10 +214,16 @@ export default class MdToHtml { } private pluginOptions(name: string) { + // Currently link_close is only used to append the media player to + // the resource links so we use the mediaPlayers plugin options for + // it. + if (name === 'link_close') name = 'mediaPlayers'; + let o = this.pluginOptions_[name] ? this.pluginOptions_[name] : {}; o = Object.assign({ enabled: true, }, o); + return o; } @@ -348,6 +367,10 @@ export default class MdToHtml { codeTheme: 'atom-one-light.css', theme: Object.assign({}, defaultNoteStyle, theme), plugins: {}, + + audioPlayerEnabled: this.pluginEnabled('audioPlayer'), + videoPlayerEnabled: this.pluginEnabled('videoPlayer'), + pdfViewerEnabled: this.pluginEnabled('pdfViewer'), }, options); // The "codeHighlightCacheKey" option indicates what set of cached object should be @@ -373,6 +396,7 @@ export default class MdToHtml { pluginAssets: {}, cache: this.contextCache_, userData: {}, + currentLinks: [], }; const markdownIt = new MarkdownIt({ diff --git a/packages/renderer/MdToHtml/linkReplacement.test.ts b/packages/renderer/MdToHtml/linkReplacement.test.ts index 58749d568..60ddad1d7 100644 --- a/packages/renderer/MdToHtml/linkReplacement.test.ts +++ b/packages/renderer/MdToHtml/linkReplacement.test.ts @@ -3,12 +3,12 @@ import linkReplacement from './linkReplacement'; describe('linkReplacement', () => { test('should handle non-resource links', () => { - const r = linkReplacement('https://example.com/test'); + const r = linkReplacement('https://example.com/test').html; expect(r).toBe(''); }); test('should handle non-resource links - simple rendering', () => { - const r = linkReplacement('https://example.com/test', { linkRenderingType: 2 }); + const r = linkReplacement('https://example.com/test', { linkRenderingType: 2 }).html; expect(r).toBe(''); }); @@ -25,7 +25,7 @@ describe('linkReplacement', () => { }, }, }, - }); + }).html; expect(r).toBe(``); }); @@ -43,7 +43,7 @@ describe('linkReplacement', () => { }, }, }, - }); + }).html; // Since the icon is embedded as SVG, we only check for the prefix const expectedPrefix = `` + ``; + + return { + resourceReady: false, + html: `` + ``, + resource, + resourceFullPath: null, + }; } else { href = `joplin://${resourceId}`; if (resourceHrefInfo.hash) href += `#${resourceHrefInfo.hash}`; @@ -100,5 +115,10 @@ export default function(href: string, options: Options = null) { if (js) attrHtml.push(js); } - return `${icon}`; + return { + html: `${icon}`, + resourceReady: true, + resource, + resourceFullPath: resource && options?.ResourceModel?.fullPath ? options.ResourceModel.fullPath(resource) : null, + }; } diff --git a/packages/renderer/MdToHtml/renderMedia.ts b/packages/renderer/MdToHtml/renderMedia.ts new file mode 100644 index 000000000..ad8a0f2ef --- /dev/null +++ b/packages/renderer/MdToHtml/renderMedia.ts @@ -0,0 +1,41 @@ +import { Link } from '../MdToHtml'; +import { toForwardSlashes } from '../pathUtils'; +const Entities = require('html-entities').AllHtmlEntities; +const htmlentities = new Entities().encode; + +export interface Options { + audioPlayerEnabled: boolean; + videoPlayerEnabled: boolean; + pdfViewerEnabled: boolean; +} + +export default function(link: Link, options: Options) { + const resource = link.resource; + + if (!link.resourceReady || !resource || !resource.mime) return ''; + + const escapedResourcePath = htmlentities(`file://${toForwardSlashes(link.resourceFullPath)}`); + const escapedMime = htmlentities(resource.mime); + + if (options.videoPlayerEnabled && resource.mime.indexOf('video/') === 0) { + return ` + + `; + } + + if (options.audioPlayerEnabled && resource.mime.indexOf('audio/') === 0) { + return ` + + `; + } + + if (options.pdfViewerEnabled && resource.mime === 'application/pdf') { + return ``; + } + + return ''; +} diff --git a/packages/renderer/MdToHtml/rules/link_close.ts b/packages/renderer/MdToHtml/rules/link_close.ts new file mode 100644 index 000000000..f64e3a608 --- /dev/null +++ b/packages/renderer/MdToHtml/rules/link_close.ts @@ -0,0 +1,22 @@ +// This rule is used to add a media player for certain resource types below +// the link. + +import { RuleOptions } from '../../MdToHtml'; +import renderMedia, { Options as RenderMediaOptions } from '../renderMedia'; + +function plugin(markdownIt: any, ruleOptions: RuleOptions) { + const defaultRender = markdownIt.renderer.rules.link_close || function(tokens: any, idx: any, options: any, _env: any, self: any) { + return self.renderToken(tokens, idx, options); + }; + + markdownIt.renderer.rules.link_close = function(tokens: any[], idx: number, options: any, env: any, self: any) { + const defaultOutput = defaultRender(tokens, idx, options, env, self); + const link = ruleOptions.context.currentLinks.pop(); + + if (!link || ruleOptions.linkRenderingType === 2 || ruleOptions.plainResourceRendering) return defaultOutput; + + return [defaultOutput, renderMedia(link, ruleOptions as RenderMediaOptions)].join(''); + }; +} + +export default { plugin }; diff --git a/packages/renderer/MdToHtml/rules/link_open.ts b/packages/renderer/MdToHtml/rules/link_open.ts index 0a3c92d64..69ee610b8 100644 --- a/packages/renderer/MdToHtml/rules/link_open.ts +++ b/packages/renderer/MdToHtml/rules/link_open.ts @@ -12,7 +12,7 @@ function plugin(markdownIt: any, ruleOptions: RuleOptions) { const isResourceUrl = ruleOptions.resources && !!resourceHrefInfo; const title = utils.getAttr(token.attrs, 'title', isResourceUrl ? '' : href); - return linkReplacement(href, { + const replacement = linkReplacement(href, { title, resources: ruleOptions.resources, ResourceModel: ruleOptions.ResourceModel, @@ -21,6 +21,15 @@ function plugin(markdownIt: any, ruleOptions: RuleOptions) { postMessageSyntax: ruleOptions.postMessageSyntax, enableLongPress: ruleOptions.enableLongPress, }); + + ruleOptions.context.currentLinks.push({ + href: href, + resource: replacement.resource, + resourceReady: replacement.resourceReady, + resourceFullPath: replacement.resourceFullPath, + }); + + return replacement.html; }; } diff --git a/packages/renderer/noteStyle.ts b/packages/renderer/noteStyle.ts index 917e89b27..4f5ba044c 100644 --- a/packages/renderer/noteStyle.ts +++ b/packages/renderer/noteStyle.ts @@ -333,6 +333,15 @@ export default function(theme: any) { pointer-events: none; } + .media-player { + width: 100%; + margin-top: 10px; + } + + .media-player.media-pdf { + min-height: 100vh; + } + /* Clear the CODE style if the element is within a joplin-editable block */ .mce-content-body .joplin-editable code { border: none; diff --git a/packages/renderer/pathUtils.ts b/packages/renderer/pathUtils.ts index 227168c77..ba8a4b43e 100644 --- a/packages/renderer/pathUtils.ts +++ b/packages/renderer/pathUtils.ts @@ -28,3 +28,7 @@ export function fileExtension(path: string) { if (output.length <= 1) return ''; return output[output.length - 1]; } + +export function toForwardSlashes(path: string) { + return path.replace(/\\/g, '/'); +}