self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

Clipper: add options to make fetchBlob exit faster, if needed (#9252)

Browse Source 
Co-authored-by: Laurent Cozic <laurent22@users.noreply.github.com>
This commit is contained in:
pedr 2023-11-19 07:44:27 -03:00 committed by GitHub
parent dc20402f5b
commit 4aeb2fafb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
packages/lib/services/rest/routes/notes.ts
View File

@ -63,6 +63,11 @@ type RequestNote = {
stylesheets: any; stylesheets: any;
}; };
type FetchOptions = {
timeout?: number;
maxRedirects?: number;
};
async function requestNoteToNote(requestNote: RequestNote): Promise<NoteEntity> { async function requestNoteToNote(requestNote: RequestNote): Promise<NoteEntity> {
const output: any = { const output: any = {
title: requestNote.title ? requestNote.title : '', title: requestNote.title ? requestNote.title : '',
@ -184,7 +189,7 @@ async function tryToGuessExtFromMimeType(response: any, mediaPath: string) {
return newMediaPath; return newMediaPath;
} }
export async function downloadMediaFile(url: string /* , allowFileProtocolImages */) { export async function downloadMediaFile(url: string, fetchOptions?: FetchOptions) {
logger.info('Downloading media file', url); logger.info('Downloading media file', url);
const tempDir = Setting.value('tempDir'); const tempDir = Setting.value('tempDir');
@ -225,7 +230,7 @@ export async function downloadMediaFile(url: string /* , allowFileProtocolImages
const localPath = fileUriToPath(url); const localPath = fileUriToPath(url);
await shim.fsDriver().copy(localPath, mediaPath); await shim.fsDriver().copy(localPath, mediaPath);
} else { } else {
const response = await shim.fetchBlob(url, { path: mediaPath, maxRetry: 1 }); const response = await shim.fetchBlob(url, { path: mediaPath, maxRetry: 1, ...fetchOptions });
// If we could not find the file extension from the URL, try to get it // If we could not find the file extension from the URL, try to get it
// now based on the Content-Type header. // now based on the Content-Type header.
@ -238,13 +243,13 @@ export async function downloadMediaFile(url: string /* , allowFileProtocolImages
} }
} }
async function downloadMediaFiles(urls: string[] /* , allowFileProtocolImages:boolean */) { async function downloadMediaFiles(urls: string[], fetchOptions?: FetchOptions) {
const PromisePool = require('es6-promise-pool'); const PromisePool = require('es6-promise-pool');
const output: any = {}; const output: any = {};
const downloadOne = async (url: string) => { const downloadOne = async (url: string) => {
const mediaPath = await downloadMediaFile(url); // , allowFileProtocolImages); const mediaPath = await downloadMediaFile(url, fetchOptions); // , allowFileProtocolImages);
if (mediaPath) output[url] = { path: mediaPath, originalUrl: url }; if (mediaPath) output[url] = { path: mediaPath, originalUrl: url };
}; };
@ -369,14 +374,14 @@ async function attachImageFromDataUrl(note: any, imageDataUrl: string, cropRect:
return await shim.attachFileToNote(note, tempFilePath); return await shim.attachFileToNote(note, tempFilePath);
} }
export const extractNoteFromHTML = async (requestNote: RequestNote, requestId: number, imageSizes: any) => { export const extractNoteFromHTML = async (requestNote: RequestNote, requestId: number, imageSizes: any, fetchOptions?: FetchOptions) => {
const note = await requestNoteToNote(requestNote); const note = await requestNoteToNote(requestNote);
const mediaUrls = extractMediaUrls(note.markup_language, note.body); const mediaUrls = extractMediaUrls(note.markup_language, note.body);
logger.info(`Request (${requestId}): Downloading media files: ${mediaUrls.length}`); logger.info(`Request (${requestId}): Downloading media files: ${mediaUrls.length}`);
const mediaFiles = await downloadMediaFiles(mediaUrls); // , allowFileProtocolImages); const mediaFiles = await downloadMediaFiles(mediaUrls, fetchOptions); // , allowFileProtocolImages);
logger.info(`Request (${requestId}): Creating resources from paths: ${Object.getOwnPropertyNames(mediaFiles).length}`); logger.info(`Request (${requestId}): Creating resources from paths: ${Object.getOwnPropertyNames(mediaFiles).length}`);

11
packages/lib/shim-init-node.js
View File

@ -460,6 +460,11 @@ function shimInit(options = null) {
if (!options.method) options.method = 'GET'; if (!options.method) options.method = 'GET';
// if (!('maxRetry' in options)) options.maxRetry = 5; // if (!('maxRetry' in options)) options.maxRetry = 5;
// 21 maxRedirects is the default amount from follow-redirects library
// 20 seems to be the max amount that most popular browsers will allow
if (!options.maxRedirects) options.maxRedirects = 21;
if (!options.timeout) options.timeout = undefined;
const urlParse = require('url').parse; const urlParse = require('url').parse;
url = urlParse(url.trim()); url = urlParse(url.trim());
@ -490,6 +495,8 @@ function shimInit(options = null) {
method: method, method: method,
path: url.pathname + (url.query ? `?${url.query}` : ''), path: url.pathname + (url.query ? `?${url.query}` : ''),
headers: headers, headers: headers,
timeout: options.timeout,
maxRedirects: options.maxRedirects,
}; };
const resolvedProxyUrl = resolveProxyUrl(proxySettings.proxyUrl); const resolvedProxyUrl = resolveProxyUrl(proxySettings.proxyUrl);
@ -551,6 +558,10 @@ function shimInit(options = null) {
}); });
}); });
request.on('timeout', () => {
request.destroy(new Error(`Request timed out. Timeout value: ${requestOptions.timeout}ms.`));
});
request.on('error', (error) => { request.on('error', (error) => {
cleanUpOnError(error); cleanUpOnError(error);
}); });