2023-12-15 15:28:09 +02:00
import shim , { CreateResourceFromPathOptions } from './shim' ;
2023-12-13 21:24:58 +02:00
import GeolocationNode from './geolocation-node' ;
import { setLocale , defaultLocale , closestSupportedLocale } from './locale' ;
import FsDriverNode from './fs-driver-node' ;
import Note from './models/Note' ;
import Resource from './models/Resource' ;
import { basename , fileExtension , safeFileExtension } from './path-utils' ;
import * as fs from 'fs-extra' ;
import * as pdfJsNamespace from 'pdfjs-dist' ;
import { writeFile } from 'fs/promises' ;
2023-12-15 15:28:09 +02:00
import { ResourceEntity } from './services/database/types' ;
2024-02-03 00:59:15 +02:00
import { TextItem } from 'pdfjs-dist/types/src/display/api' ;
2024-02-06 18:24:00 +02:00
import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters' ;
2021-01-27 19:42:58 +02:00
2022-07-10 16:26:24 +02:00
const { FileApiDriverLocal } = require ( './file-api-driver-local' ) ;
2020-11-05 18:58:23 +02:00
const mimeUtils = require ( './mime-utils.js' ) . mime ;
const { _ } = require ( './locale' ) ;
2020-02-27 02:14:40 +02:00
const http = require ( 'http' ) ;
const https = require ( 'https' ) ;
2022-07-10 15:54:31 +02:00
const { HttpProxyAgent , HttpsProxyAgent } = require ( 'hpagent' ) ;
2020-03-10 01:24:57 +02:00
const toRelative = require ( 'relative' ) ;
2020-10-09 19:35:46 +02:00
const timers = require ( 'timers' ) ;
2020-10-28 17:47:36 +02:00
const zlib = require ( 'zlib' ) ;
2021-11-17 14:54:34 +02:00
const dgram = require ( 'dgram' ) ;
2020-10-28 17:47:36 +02:00
2023-12-13 21:24:58 +02:00
const proxySettings : any = { } ;
2022-07-10 15:54:31 +02:00
2023-12-13 21:24:58 +02:00
function fileExists ( filePath : string ) {
2020-10-28 17:47:36 +02:00
try {
return fs . statSync ( filePath ) . isFile ( ) ;
2023-02-16 12:55:24 +02:00
} catch ( error ) {
2020-10-28 17:47:36 +02:00
return false ;
}
}
2023-12-13 21:24:58 +02:00
function isUrlHttps ( url : string ) {
2022-07-10 15:54:31 +02:00
return url . startsWith ( 'https' ) ;
}
2023-12-13 21:24:58 +02:00
function resolveProxyUrl ( proxyUrl : string ) {
2022-07-10 15:54:31 +02:00
return (
proxyUrl ||
process . env [ 'http_proxy' ] ||
process . env [ 'https_proxy' ] ||
process . env [ 'HTTP_PROXY' ] ||
process . env [ 'HTTPS_PROXY' ]
) ;
}
2021-01-27 19:42:58 +02:00
// https://github.com/sindresorhus/callsites/blob/main/index.js
function callsites() {
const _prepareStackTrace = Error . prepareStackTrace ;
Error . prepareStackTrace = ( _any , stack ) = > stack ;
const stack = new Error ( ) . stack . slice ( 1 ) ;
Error . prepareStackTrace = _prepareStackTrace ;
return stack ;
}
2023-12-13 21:24:58 +02:00
const gunzipFile = function ( source : string , destination : string ) {
2020-10-28 17:47:36 +02:00
if ( ! fileExists ( source ) ) {
throw new Error ( ` No such file: ${ source } ` ) ;
}
return new Promise ( ( resolve , reject ) = > {
// prepare streams
const src = fs . createReadStream ( source ) ;
const dest = fs . createWriteStream ( destination ) ;
// extract the archive
src . pipe ( zlib . createGunzip ( ) ) . pipe ( dest ) ;
// callback on extract completion
2023-02-20 17:02:29 +02:00
dest . on ( 'close' , ( ) = > {
2023-12-13 21:24:58 +02:00
resolve ( null ) ;
2020-10-28 17:47:36 +02:00
} ) ;
src . on ( 'error' , ( ) = > {
reject ( ) ;
} ) ;
dest . on ( 'error' , ( ) = > {
reject ( ) ;
} ) ;
} ) ;
} ;
2017-07-24 20:01:40 +02:00
2023-12-13 21:24:58 +02:00
function setupProxySettings ( options : any ) {
2022-07-10 15:54:31 +02:00
proxySettings . maxConcurrentConnections = options . maxConcurrentConnections ;
proxySettings . proxyTimeout = options . proxyTimeout ;
proxySettings . proxyEnabled = options . proxyEnabled ;
proxySettings . proxyUrl = options . proxyUrl ;
}
2023-12-13 21:24:58 +02:00
interface ShimInitOptions {
sharp : any ;
keytar : any ;
React : any ;
2024-01-26 12:32:35 +02:00
appVersion : ( ) = > string ;
2023-12-13 21:24:58 +02:00
electronBridge : any ;
nodeSqlite : any ;
pdfJs : typeof pdfJsNamespace ;
}
function shimInit ( options : ShimInitOptions = null ) {
2021-10-01 20:35:27 +02:00
options = {
sharp : null ,
keytar : null ,
React : null ,
appVersion : null ,
electronBridge : null ,
nodeSqlite : null ,
2023-12-13 21:24:58 +02:00
pdfJs : null ,
2021-10-01 20:35:27 +02:00
. . . options ,
} ;
const sharp = options . sharp ;
const keytar = ( shim . isWindows ( ) || shim . isMac ( ) ) && ! shim . isPortable ( ) ? options.keytar : null ;
const appVersion = options . appVersion ;
2023-12-13 21:24:58 +02:00
const pdfJs = options . pdfJs ;
2022-07-10 15:54:31 +02:00
2021-10-01 20:35:27 +02:00
shim . setNodeSqlite ( options . nodeSqlite ) ;
2020-11-05 18:58:23 +02:00
2019-07-29 15:43:53 +02:00
shim . fsDriver = ( ) = > {
throw new Error ( 'Not implemented' ) ;
} ;
2017-07-24 20:01:40 +02:00
shim . FileApiDriverLocal = FileApiDriverLocal ;
2017-07-10 20:09:58 +02:00
shim . Geolocation = GeolocationNode ;
2017-07-24 20:01:40 +02:00
shim . FormData = require ( 'form-data' ) ;
2020-11-05 18:58:23 +02:00
shim . sjclModule = require ( './vendor/sjcl.js' ) ;
2021-10-01 20:35:27 +02:00
shim . electronBridge_ = options . electronBridge ;
2017-10-15 13:13:09 +02:00
2018-01-21 19:01:37 +02:00
shim . fsDriver = ( ) = > {
if ( ! shim . fsDriver_ ) shim . fsDriver_ = new FsDriverNode ( ) ;
return shim . fsDriver_ ;
2019-07-29 15:43:53 +02:00
} ;
2018-01-21 19:01:37 +02:00
2021-11-17 14:54:34 +02:00
shim . dgram = ( ) = > {
return dgram ;
} ;
2021-10-01 20:35:27 +02:00
if ( options . React ) {
2020-11-19 18:38:44 +02:00
shim . react = ( ) = > {
2021-10-01 20:35:27 +02:00
return options . React ;
2020-11-19 18:38:44 +02:00
} ;
}
2021-10-01 20:35:27 +02:00
shim . electronBridge = ( ) = > {
return shim . electronBridge_ ;
} ;
2020-05-21 10:14:33 +02:00
shim . randomBytes = async count = > {
2017-12-12 19:51:07 +02:00
const buffer = require ( 'crypto' ) . randomBytes ( count ) ;
return Array . from ( buffer ) ;
2019-07-29 15:43:53 +02:00
} ;
2017-12-12 19:51:07 +02:00
2023-12-13 21:24:58 +02:00
shim . detectAndSetLocale = function ( Setting : any ) {
2023-05-10 12:05:55 +02:00
let locale = shim . isElectron ( ) ? shim . electronBridge ( ) . getLocale ( ) : process . env . LANG ;
2017-11-04 14:23:46 +02:00
if ( ! locale ) locale = defaultLocale ( ) ;
locale = locale . split ( '.' ) ;
locale = locale [ 0 ] ;
locale = closestSupportedLocale ( locale ) ;
Setting . setValue ( 'locale' , locale ) ;
setLocale ( locale ) ;
return locale ;
2019-07-29 15:43:53 +02:00
} ;
2017-11-04 14:23:46 +02:00
2018-05-10 11:45:44 +02:00
shim . writeImageToFile = async function ( nativeImage , mime , targetPath ) {
2019-07-29 15:43:53 +02:00
if ( shim . isElectron ( ) ) {
// For Electron
2018-05-25 09:51:54 +02:00
let buffer = null ;
2018-05-10 11:45:44 +02:00
2018-05-25 09:51:54 +02:00
mime = mime . toLowerCase ( ) ;
2018-05-10 11:45:44 +02:00
2018-05-25 09:51:54 +02:00
if ( mime === 'image/png' ) {
buffer = nativeImage . toPNG ( ) ;
} else if ( mime === 'image/jpg' || mime === 'image/jpeg' ) {
buffer = nativeImage . toJPEG ( 90 ) ;
}
2018-05-10 11:45:44 +02:00
2019-09-19 23:51:18 +02:00
if ( ! buffer ) throw new Error ( ` Cannot resize image because mime type " ${ mime } " is not supported: ${ targetPath } ` ) ;
2018-05-10 11:45:44 +02:00
2018-05-25 09:51:54 +02:00
await shim . fsDriver ( ) . writeFile ( targetPath , buffer , 'buffer' ) ;
} else {
throw new Error ( 'Node support not implemented' ) ;
}
2019-07-29 15:43:53 +02:00
} ;
2018-05-10 11:45:44 +02:00
2020-03-31 23:40:38 +02:00
shim . showMessageBox = ( message , options = null ) = > {
if ( shim . isElectron ( ) ) {
2021-10-01 20:35:27 +02:00
return shim . electronBridge ( ) . showMessageBox ( message , options ) ;
2020-03-31 23:40:38 +02:00
} else {
throw new Error ( 'Not implemented' ) ;
}
} ;
2023-12-13 21:24:58 +02:00
const handleResizeImage_ = async function ( filePath : string , targetPath : string , mime : string , resizeLargeImages : string ) {
2019-05-12 12:38:33 +02:00
const maxDim = Resource . IMAGE_MAX_DIMENSION ;
2019-07-29 15:43:53 +02:00
if ( shim . isElectron ( ) ) {
2024-02-02 19:58:27 +02:00
// For Electron/renderer process
// Note that we avoid nativeImage because it loses rotation metadata.
// See https://github.com/electron/electron/issues/41189
//
// After the upstream bug has been fixed, this should be reverted to using
// nativeImage (see commit 99e8818ba093a931b1a0cbccbee0b94a4fd37a54 for the
// original code).
const image = new Image ( ) ;
image . src = filePath ;
await new Promise < void > ( ( resolve , reject ) = > {
image . onload = ( ) = > resolve ( ) ;
image . onerror = ( ) = > reject ( ` Image at ${ filePath } failed to load. ` ) ;
image . onabort = ( ) = > reject ( ` Loading stopped for image at ${ filePath } . ` ) ;
} ) ;
if ( ! image . complete || ( image . width === 0 && image . height === 0 ) ) {
throw new Error ( ` Image is invalid or does not exist: ${ filePath } ` ) ;
}
2018-04-22 21:10:43 +02:00
2023-08-08 16:49:54 +02:00
const saveOriginalImage = async ( ) = > {
2021-08-05 16:08:57 +02:00
await shim . fsDriver ( ) . copy ( filePath , targetPath ) ;
2020-03-31 23:40:38 +02:00
return true ;
2023-08-08 16:49:54 +02:00
} ;
const saveResizedImage = async ( ) = > {
2024-02-02 19:58:27 +02:00
let newWidth , newHeight ;
if ( image . width > image . height ) {
newWidth = maxDim ;
newHeight = image . height * maxDim / image . width ;
2023-08-08 16:49:54 +02:00
} else {
2024-02-02 19:58:27 +02:00
newWidth = image . width * maxDim / image . height ;
newHeight = maxDim ;
2023-08-08 16:49:54 +02:00
}
2024-02-02 19:58:27 +02:00
const canvas = new OffscreenCanvas ( newWidth , newHeight ) ;
const ctx = canvas . getContext ( '2d' ) ;
ctx . drawImage ( image , 0 , 0 , newWidth , newHeight ) ;
const resizedImage = await canvas . convertToBlob ( { type : mime } ) ;
await fs . writeFile ( targetPath , Buffer . from ( await resizedImage . arrayBuffer ( ) ) ) ;
2023-08-08 16:49:54 +02:00
return true ;
} ;
2024-02-02 19:58:27 +02:00
const canResize = image . width > maxDim || image . height > maxDim ;
2023-08-08 16:49:54 +02:00
if ( canResize ) {
if ( resizeLargeImages === 'alwaysAsk' ) {
const Yes = 0 , No = 1 , Cancel = 2 ;
2024-02-02 19:58:27 +02:00
const userAnswer = shim . showMessageBox ( ` ${ _ ( 'You are about to attach a large image (%dx%d pixels). Would you like to resize it down to %d pixels before attaching it?' , image . width , image . height , maxDim ) } \ n \ n ${ _ ( '(You may disable this prompt in the options)' ) } ` , {
2023-08-08 16:49:54 +02:00
buttons : [ _ ( 'Yes' ) , _ ( 'No' ) , _ ( 'Cancel' ) ] ,
} ) ;
if ( userAnswer === Yes ) return await saveResizedImage ( ) ;
if ( userAnswer === No ) return await saveOriginalImage ( ) ;
if ( userAnswer === Cancel ) return false ;
} else if ( resizeLargeImages === 'alwaysResize' ) {
return await saveResizedImage ( ) ;
}
2018-04-22 21:10:43 +02:00
}
2023-08-08 16:49:54 +02:00
return await saveOriginalImage ( ) ;
2019-07-29 15:43:53 +02:00
} else {
// For the CLI tool
2019-05-12 12:38:33 +02:00
const image = sharp ( filePath ) ;
const md = await image . metadata ( ) ;
if ( md . width <= maxDim && md . height <= maxDim ) {
2023-12-13 21:24:58 +02:00
await shim . fsDriver ( ) . copy ( filePath , targetPath ) ;
2020-03-31 23:40:38 +02:00
return true ;
2019-05-12 12:38:33 +02:00
}
2018-04-22 21:10:43 +02:00
return new Promise ( ( resolve , reject ) = > {
2019-07-29 15:43:53 +02:00
image
. resize ( Resource . IMAGE_MAX_DIMENSION , Resource . IMAGE_MAX_DIMENSION , {
fit : 'inside' ,
withoutEnlargement : true ,
} )
2023-12-13 21:24:58 +02:00
. toFile ( targetPath , ( error : any , info : any ) = > {
2023-02-16 12:55:24 +02:00
if ( error ) {
reject ( error ) ;
2019-07-29 15:43:53 +02:00
} else {
resolve ( info ) ;
}
} ) ;
2017-11-11 00:18:00 +02:00
} ) ;
2018-04-22 21:10:43 +02:00
}
2019-07-29 15:43:53 +02:00
} ;
2017-11-11 00:18:00 +02:00
2022-03-28 17:35:41 +02:00
// This is a bit of an ugly method that's used to both create a new resource
// from a file, and update one. To update a resource, pass the
// destinationResourceId option. This method is indirectly tested in
// Api.test.ts.
2023-12-15 15:28:09 +02:00
shim . createResourceFromPath = async function ( filePath , defaultProps : ResourceEntity = null , options : CreateResourceFromPathOptions = null ) {
options = {
resizeLargeImages : 'always' , // 'always', 'ask' or 'never'
2020-07-24 02:45:15 +02:00
userSideValidation : false ,
2023-12-15 15:28:09 +02:00
destinationResourceId : '' ,
. . . options ,
} ;
2020-03-31 23:40:38 +02:00
2018-05-23 15:25:59 +02:00
const readChunk = require ( 'read-chunk' ) ;
const imageType = require ( 'image-type' ) ;
2022-04-11 18:01:01 +02:00
const isUpdate = ! ! options . destinationResourceId ;
2020-11-05 18:58:23 +02:00
const uuid = require ( './uuid' ) . default ;
2017-11-11 00:18:00 +02:00
if ( ! ( await fs . pathExists ( filePath ) ) ) throw new Error ( _ ( 'Cannot access %s' , filePath ) ) ;
2019-02-03 20:58:44 +02:00
defaultProps = defaultProps ? defaultProps : { } ;
2022-03-28 17:35:41 +02:00
let resourceId = defaultProps . id ? defaultProps.id : uuid.create ( ) ;
2022-04-11 18:01:01 +02:00
if ( isUpdate ) resourceId = options . destinationResourceId ;
2019-02-03 20:58:44 +02:00
2022-04-11 18:01:01 +02:00
let resource = isUpdate ? { } : Resource . new ( ) ;
2019-02-03 20:58:44 +02:00
resource . id = resourceId ;
2022-04-11 18:01:01 +02:00
// When this is an update we auto-update the mime type, in case the
// content type has changed, but we keep the title. It is still possible
// to modify the title on update using defaultProps.
2019-10-08 21:36:33 +02:00
resource . mime = mimeUtils . fromFilename ( filePath ) ;
2022-04-11 18:01:01 +02:00
if ( ! isUpdate ) resource . title = basename ( filePath ) ;
2017-12-02 01:15:49 +02:00
2018-05-23 15:25:59 +02:00
let fileExt = safeFileExtension ( fileExtension ( filePath ) ) ;
if ( ! resource . mime ) {
const buffer = await readChunk ( filePath , 0 , 64 ) ;
const detectedType = imageType ( buffer ) ;
if ( detectedType ) {
fileExt = detectedType . ext ;
resource . mime = detectedType . mime ;
} else {
resource . mime = 'application/octet-stream' ;
}
}
resource . file_extension = fileExt ;
2017-11-11 00:18:00 +02:00
2020-03-14 01:46:14 +02:00
const targetPath = Resource . fullPath ( resource ) ;
2017-11-11 00:18:00 +02:00
2020-05-30 14:25:05 +02:00
if ( options . resizeLargeImages !== 'never' && [ 'image/jpeg' , 'image/jpg' , 'image/png' ] . includes ( resource . mime ) ) {
2020-03-31 23:40:38 +02:00
const ok = await handleResizeImage_ ( filePath , targetPath , resource . mime , options . resizeLargeImages ) ;
if ( ! ok ) return null ;
2017-11-11 00:18:00 +02:00
} else {
await fs . copy ( filePath , targetPath , { overwrite : true } ) ;
}
2020-07-24 02:45:15 +02:00
// While a whole object can be passed as defaultProps, we only just
// support the title and ID (used above). Any other prop should be
// derived from the provided file.
if ( 'title' in defaultProps ) resource . title = defaultProps . title ;
2019-02-03 20:58:44 +02:00
2019-05-12 02:15:52 +02:00
const itDoes = await shim . fsDriver ( ) . waitTillExists ( targetPath ) ;
2019-09-19 23:51:18 +02:00
if ( ! itDoes ) throw new Error ( ` Resource file was not created: ${ targetPath } ` ) ;
2019-05-12 02:15:52 +02:00
2019-05-11 18:55:40 +02:00
const fileStat = await shim . fsDriver ( ) . stat ( targetPath ) ;
resource . size = fileStat . size ;
2023-12-13 21:24:58 +02:00
const saveOptions : any = { isNew : true } ;
2020-07-24 02:45:15 +02:00
if ( options . userSideValidation ) saveOptions . userSideValidation = true ;
2022-03-28 17:35:41 +02:00
2022-04-11 18:01:01 +02:00
if ( isUpdate ) {
2022-03-28 17:35:41 +02:00
saveOptions . isNew = false ;
const tempPath = ` ${ targetPath } .tmp ` ;
await shim . fsDriver ( ) . move ( targetPath , tempPath ) ;
resource = await Resource . save ( resource , saveOptions ) ;
await Resource . updateResourceBlobContent ( resource . id , tempPath ) ;
await shim . fsDriver ( ) . remove ( tempPath ) ;
return resource ;
} else {
return Resource . save ( resource , saveOptions ) ;
}
2019-07-29 15:43:53 +02:00
} ;
2018-05-23 13:14:38 +02:00
2020-05-02 17:41:07 +02:00
shim . attachFileToNoteBody = async function ( noteBody , filePath , position = null , options = null ) {
2023-10-31 18:53:47 +02:00
options = { createFileURL : false , markupLanguage : 1 , . . . options } ;
2020-03-31 23:40:38 +02:00
2019-07-29 12:16:47 +02:00
const { basename } = require ( 'path' ) ;
2020-11-05 18:58:23 +02:00
const { escapeTitleText } = require ( './markdownUtils' ) . default ;
const { toFileProtocolPath } = require ( './path-utils' ) ;
2019-07-29 12:16:47 +02:00
2020-03-31 23:40:38 +02:00
let resource = null ;
if ( ! options . createFileURL ) {
resource = await shim . createResourceFromPath ( filePath , null , options ) ;
2020-04-03 00:01:14 +02:00
if ( ! resource ) return null ;
2019-07-29 12:16:47 +02:00
}
2018-05-23 13:14:38 +02:00
2018-02-25 19:01:16 +02:00
const newBody = [ ] ;
2018-05-10 11:45:44 +02:00
if ( position === null ) {
2020-05-02 17:41:07 +02:00
position = noteBody ? noteBody.length : 0 ;
2018-05-10 11:45:44 +02:00
}
2020-05-02 17:41:07 +02:00
if ( noteBody && position ) newBody . push ( noteBody . substr ( 0 , position ) ) ;
2019-07-29 12:16:47 +02:00
2020-03-31 23:40:38 +02:00
if ( ! options . createFileURL ) {
2023-10-31 18:53:47 +02:00
newBody . push ( Resource . markupTag ( resource , options . markupLanguage ) ) ;
2019-07-29 12:16:47 +02:00
} else {
2020-06-07 13:55:40 +02:00
const filename = escapeTitleText ( basename ( filePath ) ) ; // to get same filename as standard drag and drop
2020-03-14 01:46:14 +02:00
const fileURL = ` [ ${ filename } ]( ${ toFileProtocolPath ( filePath ) } ) ` ;
2019-07-29 12:16:47 +02:00
newBody . push ( fileURL ) ;
}
2020-05-02 17:41:07 +02:00
if ( noteBody ) newBody . push ( noteBody . substr ( position ) ) ;
return newBody . join ( '\n\n' ) ;
} ;
2023-12-13 21:24:58 +02:00
shim . attachFileToNote = async function ( note , filePath , position : number = null , options : any = null ) {
2023-11-01 16:58:21 +02:00
if ( ! options ) options = { } ;
2023-10-31 18:53:47 +02:00
if ( note . markup_language ) options . markupLanguage = note . markup_language ;
2020-05-02 17:41:07 +02:00
const newBody = await shim . attachFileToNoteBody ( note . body , filePath , position , options ) ;
if ( ! newBody ) return null ;
2018-02-25 19:01:16 +02:00
2023-06-01 13:02:36 +02:00
const newNote = { . . . note , body : newBody } ;
2020-11-10 17:59:30 +02:00
return Note . save ( newNote ) ;
2019-07-29 15:43:53 +02:00
} ;
2017-11-11 00:18:00 +02:00
2022-02-06 18:42:00 +02:00
shim . imageToDataUrl = async ( filePath , maxSize ) = > {
if ( shim . isElectron ( ) ) {
2022-02-11 21:53:42 +02:00
const nativeImage = require ( 'electron' ) . nativeImage ;
2022-02-11 21:08:20 +02:00
let image = nativeImage . createFromPath ( filePath ) ;
2022-02-06 18:42:00 +02:00
if ( ! image ) throw new Error ( ` Could not load image: ${ filePath } ` ) ;
2022-02-07 19:23:20 +02:00
const ext = fileExtension ( filePath ) . toLowerCase ( ) ;
if ( ! [ 'jpg' , 'jpeg' , 'png' ] . includes ( ext ) ) throw new Error ( ` Unsupported file format: ${ ext } ` ) ;
2022-02-06 18:42:00 +02:00
if ( maxSize ) {
const size = image . getSize ( ) ;
2022-02-11 21:08:20 +02:00
if ( size . width > maxSize || size . height > maxSize ) {
console . warn ( ` Image is over ${ maxSize } px - resizing it: ${ filePath } ` ) ;
2023-12-13 21:24:58 +02:00
const options : any = { } ;
2022-02-11 21:08:20 +02:00
if ( size . width > size . height ) {
options . width = maxSize ;
} else {
options . height = maxSize ;
}
image = image . resize ( options ) ;
}
2022-02-06 18:42:00 +02:00
}
return image . toDataURL ( ) ;
} else {
throw new Error ( 'Unsupported method' ) ;
}
} ,
2018-05-25 09:51:54 +02:00
shim . imageFromDataUrl = async function ( imageDataUrl , filePath , options = null ) {
if ( options === null ) options = { } ;
if ( shim . isElectron ( ) ) {
2022-02-11 21:53:42 +02:00
const nativeImage = require ( 'electron' ) . nativeImage ;
2018-05-25 09:51:54 +02:00
let image = nativeImage . createFromDataURL ( imageDataUrl ) ;
2020-02-14 01:59:23 +02:00
if ( image . isEmpty ( ) ) throw new Error ( 'Could not convert data URL to image - perhaps the format is not supported (eg. image/gif)' ) ; // Would throw for example if the image format is no supported (eg. image/gif)
2018-09-23 19:03:11 +02:00
if ( options . cropRect ) {
// Crop rectangle values need to be rounded or the crop() call will fail
const c = options . cropRect ;
if ( 'x' in c ) c . x = Math . round ( c . x ) ;
if ( 'y' in c ) c . y = Math . round ( c . y ) ;
if ( 'width' in c ) c . width = Math . round ( c . width ) ;
if ( 'height' in c ) c . height = Math . round ( c . height ) ;
image = image . crop ( c ) ;
}
2018-05-25 09:51:54 +02:00
const mime = mimeUtils . fromDataUrl ( imageDataUrl ) ;
await shim . writeImageToFile ( image , mime , filePath ) ;
} else {
2018-09-27 10:14:05 +02:00
if ( options . cropRect ) throw new Error ( 'Crop rect not supported in Node' ) ;
const imageDataURI = require ( 'image-data-uri' ) ;
const result = imageDataURI . decode ( imageDataUrl ) ;
2019-07-29 15:43:53 +02:00
await shim . fsDriver ( ) . writeFile ( filePath , result . dataBuffer , 'buffer' ) ;
2018-05-25 09:51:54 +02:00
}
2019-07-29 15:43:53 +02:00
} ;
2018-05-25 09:51:54 +02:00
2017-10-15 13:13:09 +02:00
const nodeFetch = require ( 'node-fetch' ) ;
2018-09-27 20:35:10 +02:00
// Not used??
2020-05-21 10:14:33 +02:00
shim . readLocalFileBase64 = path = > {
2017-11-05 18:51:03 +02:00
const data = fs . readFileSync ( path ) ;
return new Buffer ( data ) . toString ( 'base64' ) ;
2019-07-29 15:43:53 +02:00
} ;
2017-11-05 18:51:03 +02:00
2022-07-10 15:54:31 +02:00
shim . fetch = async function ( url , options = { } ) {
2022-10-15 23:51:57 +02:00
try { // Check if the url is valid
new URL ( url ) ;
} catch ( error ) { // If the url is not valid, a TypeError will be thrown
throw new Error ( ` Not a valid URL: ${ url } ` ) ;
}
2022-07-10 15:54:31 +02:00
const resolvedProxyUrl = resolveProxyUrl ( proxySettings . proxyUrl ) ;
options . agent = ( resolvedProxyUrl && proxySettings . proxyEnabled ) ? shim . proxyAgent ( url , resolvedProxyUrl ) : null ;
2017-11-13 00:52:54 +02:00
return shim . fetchWithRetry ( ( ) = > {
2019-07-29 15:43:53 +02:00
return nodeFetch ( url , options ) ;
2017-11-13 00:52:54 +02:00
} , options ) ;
2019-07-29 15:43:53 +02:00
} ;
2023-12-13 21:24:58 +02:00
shim . fetchBlob = async function ( url : any , options ) {
2017-07-10 20:09:58 +02:00
if ( ! options || ! options . path ) throw new Error ( 'fetchBlob: target file path is missing' ) ;
if ( ! options . method ) options . method = 'GET' ;
2019-10-09 21:35:13 +02:00
// if (!('maxRetry' in options)) options.maxRetry = 5;
2017-07-10 20:09:58 +02:00
2023-11-19 12:44:27 +02:00
// 21 maxRedirects is the default amount from follow-redirects library
// 20 seems to be the max amount that most popular browsers will allow
if ( ! options . maxRedirects ) options . maxRedirects = 21 ;
if ( ! options . timeout ) options . timeout = undefined ;
2017-07-10 20:09:58 +02:00
const urlParse = require ( 'url' ) . parse ;
url = urlParse ( url . trim ( ) ) ;
2018-03-24 21:35:10 +02:00
const method = options . method ? options . method : 'GET' ;
2022-07-23 11:33:12 +02:00
const http = url . protocol . toLowerCase ( ) === 'http:' ? require ( 'follow-redirects' ) . http : require ( 'follow-redirects' ) . https ;
2017-07-10 20:09:58 +02:00
const headers = options . headers ? options . headers : { } ;
const filePath = options . path ;
2023-12-13 21:24:58 +02:00
function makeResponse ( response : any ) {
2017-07-10 20:09:58 +02:00
return {
ok : response.statusCode < 400 ,
path : filePath ,
2019-07-29 15:43:53 +02:00
text : ( ) = > {
return response . statusMessage ;
} ,
json : ( ) = > {
2019-09-19 23:51:18 +02:00
return { message : ` ${ response . statusCode } : ${ response . statusMessage } ` } ;
2019-07-29 15:43:53 +02:00
} ,
2017-07-10 20:09:58 +02:00
status : response.statusCode ,
headers : response.headers ,
} ;
}
2023-12-13 21:24:58 +02:00
const requestOptions : any = {
2017-07-10 20:09:58 +02:00
protocol : url.protocol ,
2018-05-20 13:20:15 +02:00
host : url.hostname ,
2017-07-10 20:09:58 +02:00
port : url.port ,
method : method ,
2019-09-19 23:51:18 +02:00
path : url.pathname + ( url . query ? ` ? ${ url . query } ` : '' ) ,
2017-07-10 20:09:58 +02:00
headers : headers ,
2023-11-19 12:44:27 +02:00
timeout : options.timeout ,
maxRedirects : options.maxRedirects ,
2017-07-10 20:09:58 +02:00
} ;
2022-07-10 15:54:31 +02:00
const resolvedProxyUrl = resolveProxyUrl ( proxySettings . proxyUrl ) ;
2022-09-05 11:42:22 +02:00
requestOptions . agent = ( resolvedProxyUrl && proxySettings . proxyEnabled ) ? shim . proxyAgent ( url . href , resolvedProxyUrl ) : null ;
2022-07-10 15:54:31 +02:00
2017-10-22 14:45:56 +02:00
const doFetchOperation = async ( ) = > {
return new Promise ( ( resolve , reject ) = > {
2023-12-13 21:24:58 +02:00
let file : any = null ;
2018-05-20 13:20:15 +02:00
2023-12-13 21:24:58 +02:00
const cleanUpOnError = ( error : any ) = > {
2018-05-20 13:20:15 +02:00
// We ignore any unlink error as we only want to report on the main error
2023-12-13 21:24:58 +02:00
void fs . unlink ( filePath )
2022-09-30 18:23:14 +02:00
// eslint-disable-next-line promise/prefer-await-to-then -- Old code before rule was applied
2019-07-29 15:43:53 +02:00
. catch ( ( ) = > { } )
2022-09-30 18:23:14 +02:00
// eslint-disable-next-line promise/prefer-await-to-then -- Old code before rule was applied
2019-07-29 15:43:53 +02:00
. then ( ( ) = > {
if ( file ) {
file . close ( ( ) = > {
file = null ;
reject ( error ) ;
} ) ;
} else {
2018-05-20 13:20:15 +02:00
reject ( error ) ;
2019-07-29 15:43:53 +02:00
}
} ) ;
} ;
2018-05-20 13:20:15 +02:00
2017-10-22 14:45:56 +02:00
try {
// Note: relative paths aren't supported
2018-05-20 13:20:15 +02:00
file = fs . createWriteStream ( filePath ) ;
2023-12-13 21:24:58 +02:00
file . on ( 'error' , ( error : any ) = > {
2018-05-20 13:20:15 +02:00
cleanUpOnError ( error ) ;
} ) ;
2017-07-10 20:09:58 +02:00
2023-12-13 21:24:58 +02:00
const request = http . request ( requestOptions , ( response : any ) = > {
2017-10-22 14:45:56 +02:00
response . pipe ( file ) ;
2017-07-10 20:09:58 +02:00
2020-10-28 17:47:36 +02:00
const isGzipped = response . headers [ 'content-encoding' ] === 'gzip' ;
2023-02-20 17:02:29 +02:00
file . on ( 'finish' , ( ) = > {
2020-10-28 17:47:36 +02:00
file . close ( async ( ) = > {
if ( isGzipped ) {
const gzipFilePath = ` ${ filePath } .gzip ` ;
await shim . fsDriver ( ) . move ( filePath , gzipFilePath ) ;
try {
await gunzipFile ( gzipFilePath , filePath ) ;
resolve ( makeResponse ( response ) ) ;
} catch ( error ) {
cleanUpOnError ( error ) ;
}
2023-12-13 21:24:58 +02:00
await shim . fsDriver ( ) . remove ( gzipFilePath ) ;
2020-10-28 17:47:36 +02:00
} else {
resolve ( makeResponse ( response ) ) ;
}
2017-10-22 14:45:56 +02:00
} ) ;
2017-07-10 20:09:58 +02:00
} ) ;
2019-07-29 15:43:53 +02:00
} ) ;
2017-07-10 20:09:58 +02:00
2023-11-19 12:44:27 +02:00
request . on ( 'timeout' , ( ) = > {
request . destroy ( new Error ( ` Request timed out. Timeout value: ${ requestOptions . timeout } ms. ` ) ) ;
} ) ;
2023-12-13 21:24:58 +02:00
request . on ( 'error' , ( error : any ) = > {
2018-05-20 13:20:15 +02:00
cleanUpOnError ( error ) ;
2017-10-22 14:45:56 +02:00
} ) ;
2018-03-24 21:35:10 +02:00
request . end ( ) ;
2018-05-20 13:20:15 +02:00
} catch ( error ) {
cleanUpOnError ( error ) ;
2017-10-22 14:45:56 +02:00
}
} ) ;
} ;
2017-11-13 00:52:54 +02:00
return shim . fetchWithRetry ( doFetchOperation , options ) ;
2019-07-29 15:43:53 +02:00
} ;
2017-12-18 22:46:22 +02:00
shim . uploadBlob = async function ( url , options ) {
2019-07-29 15:43:53 +02:00
if ( ! options || ! options . path ) throw new Error ( 'uploadBlob: source file path is missing' ) ;
2017-12-18 22:46:22 +02:00
const content = await fs . readFile ( options . path ) ;
2023-06-01 13:02:36 +02:00
options = { . . . options , body : content } ;
2017-12-18 22:46:22 +02:00
return shim . fetch ( url , options ) ;
2019-07-29 15:43:53 +02:00
} ;
2017-12-18 22:46:22 +02:00
2018-02-15 20:33:08 +02:00
shim . stringByteLength = function ( string ) {
return Buffer . byteLength ( string , 'utf-8' ) ;
2019-07-29 15:43:53 +02:00
} ;
2018-02-15 20:33:08 +02:00
2018-03-24 21:35:10 +02:00
shim . Buffer = Buffer ;
2020-05-21 10:14:33 +02:00
shim . openUrl = url = > {
2019-12-13 02:40:58 +02:00
// Returns true if it opens the file successfully; returns false if it could
// not find the file.
2021-10-01 20:35:27 +02:00
return shim . electronBridge ( ) . openExternal ( url ) ;
2019-12-13 02:40:58 +02:00
} ;
2020-02-27 02:14:40 +02:00
shim . httpAgent_ = null ;
2020-05-21 10:14:33 +02:00
shim . httpAgent = url = > {
2021-03-26 11:09:19 +02:00
if ( ! shim . httpAgent_ ) {
2020-03-14 01:46:14 +02:00
const AgentSettings = {
2020-02-27 02:14:40 +02:00
keepAlive : true ,
maxSockets : 1 ,
keepAliveMsecs : 5000 ,
} ;
2021-06-07 11:19:59 +02:00
shim . httpAgent_ = {
http : new http . Agent ( AgentSettings ) ,
https : new https . Agent ( AgentSettings ) ,
} ;
2020-02-27 02:14:40 +02:00
}
2021-06-07 11:19:59 +02:00
return url . startsWith ( 'https' ) ? shim.httpAgent_.https : shim.httpAgent_.http ;
2020-02-27 02:14:40 +02:00
} ;
2023-12-13 21:24:58 +02:00
shim . proxyAgent = ( serverUrl : string , proxyUrl : string ) = > {
2022-07-10 15:54:31 +02:00
const proxyAgentConfig = {
keepAlive : true ,
maxSockets : proxySettings.maxConcurrentConnections ,
keepAliveMsecs : 5000 ,
proxy : proxyUrl ,
timeout : proxySettings.proxyTimeout * 1000 ,
} ;
// Based on https://github.com/delvedor/hpagent#usage
if ( ! isUrlHttps ( proxyUrl ) && ! isUrlHttps ( serverUrl ) ) {
return new HttpProxyAgent ( proxyAgentConfig ) ;
} else if ( isUrlHttps ( proxyUrl ) && ! isUrlHttps ( serverUrl ) ) {
return new HttpProxyAgent ( proxyAgentConfig ) ;
} else if ( ! isUrlHttps ( proxyUrl ) && isUrlHttps ( serverUrl ) ) {
return new HttpsProxyAgent ( proxyAgentConfig ) ;
} else {
return new HttpsProxyAgent ( proxyAgentConfig ) ;
}
} ;
2019-12-13 02:40:58 +02:00
shim . openOrCreateFile = ( filepath , defaultContents ) = > {
// If the file doesn't exist, create it
if ( ! fs . existsSync ( filepath ) ) {
fs . writeFile ( filepath , defaultContents , 'utf-8' , ( error ) = > {
if ( error ) {
console . error ( ` error: ${ error } ` ) ;
}
} ) ;
}
// Open the file
2023-01-28 14:28:01 +02:00
// Don't use openUrl() there.
// The underneath require('electron').shell.openExternal() has a bug
// https://github.com/electron/electron/issues/31347
return shim . electronBridge ( ) . openItem ( filepath ) ;
2019-07-29 15:43:53 +02:00
} ;
2018-06-25 19:14:57 +02:00
2019-07-29 15:43:53 +02:00
shim . waitForFrame = ( ) = > { } ;
2020-01-24 22:56:44 +02:00
shim . appVersion = ( ) = > {
2021-04-07 19:12:37 +02:00
if ( appVersion ) return appVersion ( ) ;
// Should not happen but don't throw an error because version number is
// used in error messages.
2024-01-26 12:32:35 +02:00
return 'unknown' ;
2020-01-24 22:56:44 +02:00
} ;
2020-03-10 01:24:57 +02:00
shim . pathRelativeToCwd = ( path ) = > {
return toRelative ( process . cwd ( ) , path ) ;
} ;
2020-06-03 18:07:50 +02:00
2020-10-09 19:35:46 +02:00
shim . setTimeout = ( fn , interval ) = > {
return timers . setTimeout ( fn , interval ) ;
} ;
shim . setInterval = ( fn , interval ) = > {
return timers . setInterval ( fn , interval ) ;
} ;
shim . clearTimeout = ( id ) = > {
return timers . clearTimeout ( id ) ;
} ;
shim . clearInterval = ( id ) = > {
return timers . clearInterval ( id ) ;
} ;
2020-11-05 18:58:23 +02:00
shim . keytar = ( ) = > {
return keytar ;
} ;
2020-12-20 09:52:28 +02:00
shim . requireDynamic = ( path ) = > {
2021-01-27 19:42:58 +02:00
if ( path . indexOf ( '.' ) === 0 ) {
2023-12-13 21:24:58 +02:00
const sites : any = callsites ( ) ;
2021-01-27 19:42:58 +02:00
if ( sites . length <= 1 ) throw new Error ( ` Cannot require file (1) ${ path } ` ) ;
const filename = sites [ 1 ] . getFileName ( ) ;
if ( ! filename ) throw new Error ( ` Cannot require file (2) ${ path } ` ) ;
const fileDirName = require ( 'path' ) . dirname ( filename ) ;
return require ( ` ${ fileDirName } / ${ path } ` ) ;
} else {
return require ( path ) ;
}
2020-12-20 09:52:28 +02:00
} ;
2023-12-13 21:24:58 +02:00
2024-02-03 00:59:15 +02:00
shim . pdfExtractEmbeddedText = async ( pdfPath : string ) : Promise < string [ ] > = > {
const loadingTask = pdfJs . getDocument ( pdfPath ) ;
const doc = await loadingTask . promise ;
const textByPage = [ ] ;
2024-03-05 13:42:54 +02:00
try {
for ( let pageNum = 1 ; pageNum <= doc . numPages ; pageNum ++ ) {
const page = await doc . getPage ( pageNum ) ;
const textContent = await page . getTextContent ( ) ;
2024-02-03 00:59:15 +02:00
2024-03-05 13:42:54 +02:00
const strings = textContent . items . map ( item = > {
const text = ( item as TextItem ) . str ? ? '' ;
return text ;
} ) . join ( '\n' ) ;
2024-02-06 18:24:00 +02:00
2024-03-05 13:42:54 +02:00
// Some PDFs contain unsupported characters that can lead to hard-to-debug issues.
// We remove them here.
textByPage . push ( replaceUnsupportedCharacters ( strings ) ) ;
}
} finally {
await doc . destroy ( ) ;
2024-02-03 00:59:15 +02:00
}
return textByPage ;
} ;
2023-12-13 21:24:58 +02:00
shim . pdfToImages = async ( pdfPath : string , outputDirectoryPath : string ) : Promise < string [ ] > = > {
// We handle both the Electron app and testing framework. Potentially
// the same code could be use to support the CLI app.
const isTesting = ! shim . isElectron ( ) ;
const createCanvas = ( ) = > {
if ( isTesting ) {
return require ( 'canvas' ) . createCanvas ( ) ;
}
return document . createElement ( 'canvas' ) ;
} ;
const canvasToBuffer = async ( canvas : any ) : Promise < Buffer > = > {
if ( isTesting ) {
return canvas . toBuffer ( 'image/jpeg' , { quality : 0.8 } ) ;
} else {
const canvasToBlob = async ( canvas : HTMLCanvasElement ) : Promise < Blob > = > {
return new Promise ( resolve = > {
canvas . toBlob ( blob = > resolve ( blob ) , 'image/jpg' , 0.8 ) ;
} ) ;
} ;
const blob = await canvasToBlob ( canvas ) ;
return Buffer . from ( await blob . arrayBuffer ( ) ) ;
}
} ;
const filePrefix = ` page_ ${ Date . now ( ) } ` ;
const output : string [ ] = [ ] ;
const loadingTask = pdfJs . getDocument ( pdfPath ) ;
const doc = await loadingTask . promise ;
2024-03-05 13:42:54 +02:00
try {
for ( let pageNum = 1 ; pageNum <= doc . numPages ; pageNum ++ ) {
const page = await doc . getPage ( pageNum ) ;
const viewport = page . getViewport ( { scale : 2 } ) ;
const canvas = createCanvas ( ) ;
const ctx = canvas . getContext ( '2d' ) ;
2023-12-13 21:24:58 +02:00
2024-03-05 13:42:54 +02:00
canvas . height = viewport . height ;
canvas . width = viewport . width ;
2023-12-13 21:24:58 +02:00
2024-03-05 13:42:54 +02:00
const renderTask = page . render ( { canvasContext : ctx , viewport : viewport } ) ;
await renderTask . promise ;
2023-12-13 21:24:58 +02:00
2024-03-05 13:42:54 +02:00
const buffer = await canvasToBuffer ( canvas ) ;
const filePath = ` ${ outputDirectoryPath } / ${ filePrefix } _ ${ pageNum . toString ( ) . padStart ( 4 , '0' ) } .jpg ` ;
output . push ( filePath ) ;
await writeFile ( filePath , buffer , 'binary' ) ;
if ( ! ( await shim . fsDriver ( ) . exists ( filePath ) ) ) throw new Error ( ` Could not write to file: ${ filePath } ` ) ;
}
} finally {
await doc . destroy ( ) ;
2023-12-13 21:24:58 +02:00
}
return output ;
} ;
2017-07-10 20:09:58 +02:00
}
2022-07-10 15:54:31 +02:00
module .exports = { shimInit , setupProxySettings } ;