Desktop: Resolves #9794: Plugin API: Add support for loading PDFs with the imaging API (#10177)

packages/app-cli/tests/support/plugins/imaging/api/JoplinImaging.d.ts

@@ -1,12 +1,35 @@
 import { Rectangle } from './types';
-export interface Implementation {
-    nativeImage: any;
-}
 export interface CreateFromBufferOptions {
     width?: number;
     height?: number;
     scaleFactor?: number;
 }
+export interface CreateFromPdfOptions {
+    /**
+     * The first page to export. Defaults to `1`, the first page in
+     * the document.
+     */
+    minPage?: number;
+    /**
+     * The number of the last page to convert. Defaults to the last page
+     * if not given.
+     *
+     * If `maxPage` is greater than the number of pages in the PDF, all pages
+     * in the PDF will be converted to images.
+     */
+    maxPage?: number;
+    scaleFactor?: number;
+}
+export interface PdfInfo {
+    pageCount: number;
+}
+export interface Implementation {
+    nativeImage: {
+        createFromPath: (path: string) => Promise<any>;
+        createFromPdf: (path: string, options: CreateFromPdfOptions) => Promise<any[]>;
+    };
+    getPdfInfo: (path: string) => Promise<PdfInfo>;
+}
 export interface ResizeOptions {
     width?: number;
     height?: number;
@@ -34,9 +57,13 @@ export default class JoplinImaging {
     private cacheImage;
     createFromPath(filePath: string): Promise<Handle>;
     createFromResource(resourceId: string): Promise<Handle>;
+    createFromPdfPath(path: string, options?: CreateFromPdfOptions): Promise<Handle[]>;
+    createFromPdfResource(resourceId: string, options?: CreateFromPdfOptions): Promise<Handle[]>;
+    getPdfInfoFromPath(path: string): Promise<PdfInfo>;
+    getPdfInfoFromResource(resourceId: string): Promise<PdfInfo>;
     getSize(handle: Handle): Promise<any>;
     resize(handle: Handle, options?: ResizeOptions): Promise<string>;
-    crop(handle: Handle, rectange: Rectangle): Promise<string>;
+    crop(handle: Handle, rectangle: Rectangle): Promise<string>;
     toPngFile(handle: Handle, filePath: string): Promise<void>;
     /**
      * Quality is between 0 and 100
@@ -57,5 +84,5 @@ export default class JoplinImaging {
      * Image data is not automatically deleted by Joplin so make sure you call
      * this method on the handle once you are done.
      */
-    free(handle: Handle): Promise<void>;
+    free(handles: Handle[] | Handle): Promise<void>;
 }

packages/app-cli/tests/support/plugins/imaging/src/index.ts

@@ -1,8 +1,7 @@
 import joplin from 'api';
 import { ToolbarButtonLocation } from 'api/types';
 
-joplin.plugins.register({
+const registerMakeThumbnailCommand = async () => {
-	onStart: async function() {
 	await joplin.commands.register({
 		name: 'makeThumbnail',
 		execute: async () => {
@@ -46,5 +45,66 @@ joplin.plugins.register({
 	});
 
 	await joplin.views.toolbarButtons.create('makeThumbnailButton', 'makeThumbnail', ToolbarButtonLocation.EditorToolbar);
+};
+
+
+const registerInlinePdfCommand = async () => {
+	await joplin.commands.register({
+		name: 'inlinePdfs',
+		execute: async () => {
+			// ---------------------------------------------------------------
+			// Get the current selection & extract a resource link
+			// ---------------------------------------------------------------
+
+			const selection: string = await joplin.commands.execute('selectedText');
+
+			// Matches content of the form
+			//  [text here](:/32-letter-or-num-characters-here)
+			// Where ([a-z0-9]{32}) matches the resource ID.
+			const resourceLinkRegex = /\[.*\]\(:\/([a-z0-9]{32})\)/;
+
+			const resourceLinkMatch = selection.match(resourceLinkRegex);
+			if (!resourceLinkMatch) return;
+			const resourceId = resourceLinkMatch[1]; // The text of the region matching ([a-z0-9]{32})
+
+			const resource = await joplin.data.get(['resources', resourceId], { fields: ['mime'] });
+			const isPdf = resource.mime === 'application/pdf';
+			if (!isPdf) return;
+
+			// Clear the selection
+			await joplin.commands.execute('replaceSelection', '');
+			await joplin.commands.execute('insertText', selection);
+
+			// ---------------------------------------------------------------
+			// Convert the PDF to images
+			// ---------------------------------------------------------------
+
+			const pdfInfo = await joplin.imaging.getPdfInfoFromResource(resourceId);
+			const images = await joplin.imaging.createFromPdfResource(
+				resourceId,
+				// Convert at most 10 pages
+				{ minPage: 1, maxPage: 10, scaleFactor: 0.5 },
+			);
+
+			let pageNumber = 0;
+			for (const image of images) {
+				pageNumber++;
+				const pageResource = await joplin.imaging.toJpgResource(
+					image, { title: `Page ${pageNumber} of ${pdfInfo.pageCount}` }
+				);
+				await joplin.commands.execute('insertText', `\n- ![${pageResource.title}](:/${pageResource.id})`);
+			}
+
+			await joplin.imaging.free(images);
+		},
+	});
+
+	await joplin.views.toolbarButtons.create('inlineSelectedPdfsButton', 'inlinePdfs', ToolbarButtonLocation.EditorToolbar);
+};
+
+joplin.plugins.register({
+	onStart: async function() {
+		await registerMakeThumbnailCommand();
+		await registerInlinePdfCommand();
 	},
 });

packages/app-desktop/services/plugins/PlatformImplementation.ts

@@ -5,7 +5,10 @@ import { VersionInfo } from '@joplin/lib/services/plugins/api/types';
 import Setting from '@joplin/lib/models/Setting';
 import { reg } from '@joplin/lib/registry';
 import BasePlatformImplementation, { Joplin } from '@joplin/lib/services/plugins/BasePlatformImplementation';
-import { Implementation as ImagingImplementation } from '@joplin/lib/services/plugins/api/JoplinImaging';
+import { CreateFromPdfOptions, Implementation as ImagingImplementation } from '@joplin/lib/services/plugins/api/JoplinImaging';
+import shim from '@joplin/lib/shim';
+import { join } from 'path';
+import uuid from '@joplin/lib/uuid';
 const { clipboard, nativeImage } = require('electron');
 const packageInfo = require('../../packageInfo');
 
@@ -82,8 +85,37 @@ export default class PlatformImplementation extends BasePlatformImplementation {
 	}
 
 	public get imaging(): ImagingImplementation {
+		const createFromPdf = async (path: string, options: CreateFromPdfOptions) => {
+			const tempDir = join(Setting.value('tempDir'), uuid.createNano());
+			await shim.fsDriver().mkdir(tempDir);
+			try {
+				const paths = await shim.pdfToImages(path, tempDir, options);
+				return paths.map(path => nativeImage.createFromPath(path));
+			} finally {
+				await shim.fsDriver().remove(tempDir);
+			}
+		};
 		return {
-			nativeImage: nativeImage,
+			nativeImage: {
+				async createFromPath(path: string) {
+					if (path.toLowerCase().endsWith('.pdf')) {
+						const images = await createFromPdf(path, { minPage: 1, maxPage: 1 });
+
+						if (images.length === 0) {
+							// Match the behavior or Electron's nativeImage when reading an invalid image.
+							return nativeImage.createEmpty();
+						}
+
+						return images[0];
+					} else {
+						return nativeImage.createFromPath(path);
+					}
+				},
+				createFromPdf,
+			},
+			getPdfInfo(path: string) {
+				return shim.pdfInfo(path);
+			},
 		};
 	}
 

packages/app-mobile/plugins/PlatformImplementation.ts

@@ -75,6 +75,9 @@ export default class PlatformImplementation extends BasePlatformImplementation {
 	public get imaging(): ImagingImplementation {
 		return {
 			nativeImage: null,
+			getPdfInfo: async () => {
+				throw new Error('Not implemented: getPdfInfo');
+			},
 		};
 	}
 

packages/generator-joplin/generators/app/templates/api/JoplinImaging.d.ts

@@ -1,12 +1,35 @@
 import { Rectangle } from './types';
-export interface Implementation {
-    nativeImage: any;
-}
 export interface CreateFromBufferOptions {
     width?: number;
     height?: number;
     scaleFactor?: number;
 }
+export interface CreateFromPdfOptions {
+    /**
+     * The first page to export. Defaults to `1`, the first page in
+     * the document.
+     */
+    minPage?: number;
+    /**
+     * The number of the last page to convert. Defaults to the last page
+     * if not given.
+     *
+     * If `maxPage` is greater than the number of pages in the PDF, all pages
+     * in the PDF will be converted to images.
+     */
+    maxPage?: number;
+    scaleFactor?: number;
+}
+export interface PdfInfo {
+    pageCount: number;
+}
+export interface Implementation {
+    nativeImage: {
+        createFromPath: (path: string) => Promise<any>;
+        createFromPdf: (path: string, options: CreateFromPdfOptions) => Promise<any[]>;
+    };
+    getPdfInfo: (path: string) => Promise<PdfInfo>;
+}
 export interface ResizeOptions {
     width?: number;
     height?: number;
@@ -34,6 +57,10 @@ export default class JoplinImaging {
     private cacheImage;
     createFromPath(filePath: string): Promise<Handle>;
     createFromResource(resourceId: string): Promise<Handle>;
+    createFromPdfPath(path: string, options?: CreateFromPdfOptions): Promise<Handle[]>;
+    createFromPdfResource(resourceId: string, options?: CreateFromPdfOptions): Promise<Handle[]>;
+    getPdfInfoFromPath(path: string): Promise<PdfInfo>;
+    getPdfInfoFromResource(resourceId: string): Promise<PdfInfo>;
     getSize(handle: Handle): Promise<any>;
     resize(handle: Handle, options?: ResizeOptions): Promise<string>;
     crop(handle: Handle, rectangle: Rectangle): Promise<string>;
@@ -57,5 +84,5 @@ export default class JoplinImaging {
      * Image data is not automatically deleted by Joplin so make sure you call
      * this method on the handle once you are done.
      */
-    free(handle: Handle): Promise<void>;
+    free(handles: Handle[] | Handle): Promise<void>;
 }

packages/lib/services/plugins/api/JoplinImaging.ts

@@ -5,16 +5,43 @@ import Setting from '../../../models/Setting';
 import shim from '../../../shim';
 import { Rectangle } from './types';
 
-export interface Implementation {
-	nativeImage: any;
-}
-
 export interface CreateFromBufferOptions {
 	width?: number;
 	height?: number;
 	scaleFactor?: number;
 }
 
+export interface CreateFromPdfOptions {
+	/**
+	 * The first page to export. Defaults to `1`, the first page in
+	 * the document.
+	 */
+	minPage?: number;
+
+	/**
+	 * The number of the last page to convert. Defaults to the last page
+	 * if not given.
+	 *
+	 * If `maxPage` is greater than the number of pages in the PDF, all pages
+	 * in the PDF will be converted to images.
+	 */
+	maxPage?: number;
+
+	scaleFactor?: number;
+}
+
+export interface PdfInfo {
+	pageCount: number;
+}
+
+export interface Implementation {
+	nativeImage: {
+		createFromPath: (path: string)=> Promise<any>;
+		createFromPdf: (path: string, options: CreateFromPdfOptions)=> Promise<any[]>;
+	};
+	getPdfInfo: (path: string)=> Promise<PdfInfo>;
+}
+
 export interface ResizeOptions {
 	width?: number;
 	height?: number;
@@ -28,6 +55,14 @@ interface Image {
 	data: any;
 }
 
+const getResourcePath = async (resourceId: string): Promise<string> => {
+	const resource = await Resource.load(resourceId);
+	if (!resource) throw new Error(`No such resource: ${resourceId}`);
+	const resourcePath = await Resource.fullPath(resource);
+	if (!(await shim.fsDriver().exists(resourcePath))) throw new Error(`Could not load resource path: ${resourcePath}`);
+	return resourcePath;
+};
+
 /**
  * Provides imaging functions to resize or process images. You create an image
  * using one of the `createFrom` functions, then use the other functions to
@@ -80,15 +115,28 @@ export default class JoplinImaging {
 	// }
 
 	public async createFromPath(filePath: string): Promise<Handle> {
-		return this.cacheImage(this.implementation_.nativeImage.createFromPath(filePath));
+		return this.cacheImage(await this.implementation_.nativeImage.createFromPath(filePath));
 	}
 
 	public async createFromResource(resourceId: string): Promise<Handle> {
-		const resource = await Resource.load(resourceId);
+		return this.createFromPath(await getResourcePath(resourceId));
-		if (!resource) throw new Error(`No such resource: ${resourceId}`);
+	}
-		const resourcePath = await Resource.fullPath(resource);
+
-		if (!(await shim.fsDriver().exists(resourcePath))) throw new Error(`Could not load resource path: ${resourcePath}`);
+	public async createFromPdfPath(path: string, options?: CreateFromPdfOptions): Promise<Handle[]> {
-		return this.createFromPath(resourcePath);
+		const images = await this.implementation_.nativeImage.createFromPdf(path, options);
+		return images.map(image => this.cacheImage(image));
+	}
+
+	public async createFromPdfResource(resourceId: string, options?: CreateFromPdfOptions): Promise<Handle[]> {
+		return this.createFromPdfPath(await getResourcePath(resourceId), options);
+	}
+
+	public async getPdfInfoFromPath(path: string): Promise<PdfInfo> {
+		return await this.implementation_.getPdfInfo(path);
+	}
+
+	public async getPdfInfoFromResource(resourceId: string): Promise<PdfInfo> {
+		return this.getPdfInfoFromPath(await getResourcePath(resourceId));
 	}
 
 	public async getSize(handle: Handle) {
@@ -175,9 +223,15 @@ export default class JoplinImaging {
 	 * Image data is not automatically deleted by Joplin so make sure you call
 	 * this method on the handle once you are done.
 	 */
-	public async free(handle: Handle) {
+	public async free(handles: Handle[]|Handle) {
+		if (!Array.isArray(handles)) {
+			handles = [handles];
+		}
+
+		for (const handle of handles) {
 			const index = this.images_.findIndex(i => i.handle === handle);
 			if (index >= 0) this.images_.splice(index, 1);
 		}
+	}
 
 }

packages/lib/shim-init-node.ts

@@ -1,4 +1,4 @@
-import shim, { CreateResourceFromPathOptions } from './shim';
+import shim, { CreatePdfFromImagesOptions, CreateResourceFromPathOptions, PdfInfo } from './shim';
 import GeolocationNode from './geolocation-node';
 import { setLocale, defaultLocale, closestSupportedLocale } from './locale';
 import FsDriverNode from './fs-driver-node';
@@ -758,9 +758,13 @@ function shimInit(options: ShimInitOptions = null) {
 		}
 	};
 
+	const loadPdf = async (path: string) => {
+		const loadingTask = pdfJs.getDocument(path);
+		return await loadingTask.promise;
+	};
+
 	shim.pdfExtractEmbeddedText = async (pdfPath: string): Promise<string[]> => {
-		const loadingTask = pdfJs.getDocument(pdfPath);
+		const doc = await loadPdf(pdfPath);
-		const doc = await loadingTask.promise;
 		const textByPage = [];
 
 		try {
@@ -784,7 +788,7 @@ function shimInit(options: ShimInitOptions = null) {
 		return textByPage;
 	};
 
-	shim.pdfToImages = async (pdfPath: string, outputDirectoryPath: string): Promise<string[]> => {
+	shim.pdfToImages = async (pdfPath: string, outputDirectoryPath: string, options?: CreatePdfFromImagesOptions): Promise<string[]> => {
 		// We handle both the Electron app and testing framework. Potentially
 		// the same code could be use to support the CLI app.
 		const isTesting = !shim.isElectron();
@@ -797,12 +801,13 @@ function shimInit(options: ShimInitOptions = null) {
 		};
 
 		const canvasToBuffer = async (canvas: any): Promise<Buffer> => {
+			const quality = 0.8;
 			if (isTesting) {
-				return canvas.toBuffer('image/jpeg', { quality: 0.8 });
+				return canvas.toBuffer('image/jpeg', { quality });
 			} else {
 				const canvasToBlob = async (canvas: HTMLCanvasElement): Promise<Blob> => {
 					return new Promise(resolve => {
-						canvas.toBlob(blob => resolve(blob), 'image/jpg', 0.8);
+						canvas.toBlob(blob => resolve(blob), 'image/jpg', quality);
 					});
 				};
 
@@ -813,13 +818,14 @@ function shimInit(options: ShimInitOptions = null) {
 
 		const filePrefix = `page_${Date.now()}`;
 		const output: string[] = [];
-		const loadingTask = pdfJs.getDocument(pdfPath);
+		const doc = await loadPdf(pdfPath);
-		const doc = await loadingTask.promise;
 
 		try {
-			for (let pageNum = 1; pageNum <= doc.numPages; pageNum++) {
+			const startPage = options?.minPage ?? 1;
+			const endPage = Math.min(doc.numPages, options?.maxPage ?? doc.numPages);
+			for (let pageNum = startPage; pageNum <= endPage; pageNum++) {
 				const page = await doc.getPage(pageNum);
-				const viewport = page.getViewport({ scale: 2 });
+				const viewport = page.getViewport({ scale: options?.scaleFactor ?? 2 });
 				const canvas = createCanvas();
 				const ctx = canvas.getContext('2d');
 
@@ -841,6 +847,11 @@ function shimInit(options: ShimInitOptions = null) {
 
 		return output;
 	};
+
+	shim.pdfInfo = async (pdfPath: string): Promise<PdfInfo> => {
+		const doc = await loadPdf(pdfPath);
+		return { pageCount: doc.numPages };
+	};
 }
 
 module.exports = { shimInit, setupProxySettings };

packages/lib/shim.ts

@@ -8,6 +8,16 @@ export interface CreateResourceFromPathOptions {
 	destinationResourceId?: string;
 }
 
+export interface CreatePdfFromImagesOptions {
+	minPage?: number;
+	maxPage?: number;
+	scaleFactor?: number;
+}
+
+export interface PdfInfo {
+	pageCount: number;
+}
+
 let isTestingEnv_ = false;
 
 // We need to ensure that there's only one instance of React being used by all
@@ -282,10 +292,14 @@ const shim = {
 		throw new Error('Not implemented: textFromPdf');
 	},
 
-	pdfToImages: async (_pdfPath: string, _outputDirectoryPath: string): Promise<string[]> => {
+	pdfToImages: async (_pdfPath: string, _outputDirectoryPath: string, _options?: CreatePdfFromImagesOptions): Promise<string[]> => {
 		throw new Error('Not implemented: pdfToImages');
 	},
 
+	pdfInfo: async (_pdfPath: string): Promise<PdfInfo> => {
+		throw new Error('Not implemented: pdfInfo');
+	},
+
 	Buffer: null as any,
 
 	openUrl: (_url: string): any => {