self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-11-29 22:48:10 +02:00
Code Issues Releases Activity

Improved fetch with retry on all three platforms

Browse Source
This commit is contained in:
Laurent Cozic
2017-11-12 22:52:54 +00:00
parent 71a97bd45b
commit 2cf5234e76
10 changed files with 223 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ReactNativeClient/lib/net-utils.js
View File

@@ -1,7 +1,5 @@
const { shim } = require('lib/shim.js');
const tcpPortUsed = require('tcp-port-used');
const netUtils = {};
netUtils.ip = async () => {
@@ -15,6 +13,8 @@ netUtils.ip = async () => {
}
netUtils.findAvailablePort = async (possiblePorts, extraRandomPortsToTry = 20) => {
const tcpPortUsed = require('tcp-port-used');
for (let i = 0; i < extraRandomPortsToTry; i++) {
possiblePorts.push(Math.floor(8000 + Math.random() * 2000));
}

24
ReactNativeClient/lib/onedrive-api.js
View File

@@ -177,32 +177,8 @@ class OneDriveApi {
response = await shim.fetchBlob(url, options);
}
} catch (error) {
// let canRetry = true;
// if (error.message == 'Network request failed') {
// // Unfortunately the error 'Network request failed' doesn't have a type
// // or error code, so hopefully that message won't change and is not localized
// } else if (error.code == 'ECONNRESET') {
// // request to https://public-ch3302....1fab24cb1bd5f.md failed, reason: socket hang up"
// } else if (error.code == 'ENOTFOUND') {
// // OneDrive (or Node?) sometimes sends back a "not found" error for resources
// // that definitely exist and in this case repeating the request works.
// // Error is:
// // request to https://graph.microsoft.com/v1.0/drive/special/approot failed, reason: getaddrinfo ENOTFOUND graph.microsoft.com graph.microsoft.com:443
// } else if (error.message.indexOf('network timeout') === 0) {
// // network timeout at: https://public-ch3302...859f9b0e3ab.md
// } else {
// canRetry = false;
// }
// if (canRetry) {
// this.logger().info('Got error code ' + error.code + ': ' + error.message + ' - retrying (' + i + ')...');
// await time.sleep((i + 1) * 3);
// continue;
// } else {
this.logger().error('Got unhandled error:', error ? error.code : '', error ? error.message : '', error);
throw error;
//}
}
if (!response.ok) {

129
ReactNativeClient/lib/shim-init-node.js
View File

@@ -5,36 +5,43 @@ const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
const { time } = require('lib/time-utils.js');
const { setLocale, defaultLocale, closestSupportedLocale } = require('lib/locale.js');
function fetchRequestCanBeRetried(error) {
if (!error) return false;
// // Node requests can go wrong is so many different ways and with so
// // many different error messages... This handler inspects the error
// // and decides whether the request can safely be repeated or not.
// function fetchRequestCanBeRetried(error) {
// if (!error) return false;
// Unfortunately the error 'Network request failed' doesn't have a type
// or error code, so hopefully that message won't change and is not localized
if (error.message == 'Network request failed') return true;
// // Unfortunately the error 'Network request failed' doesn't have a type
// // or error code, so hopefully that message won't change and is not localized
// if (error.message == 'Network request failed') return true;
// request to https://public-ch3302....1fab24cb1bd5f.md failed, reason: socket hang up"
if (error.code == 'ECONNRESET') return true;
// // request to https://public-ch3302....1fab24cb1bd5f.md failed, reason: socket hang up"
// if (error.code == 'ECONNRESET') return true;
// OneDrive (or Node?) sometimes sends back a "not found" error for resources
// that definitely exist and in this case repeating the request works.
// Error is:
// request to https://graph.microsoft.com/v1.0/drive/special/approot failed, reason: getaddrinfo ENOTFOUND graph.microsoft.com graph.microsoft.com:443
if (error.code == 'ENOTFOUND') return true;
// // OneDrive (or Node?) sometimes sends back a "not found" error for resources
// // that definitely exist and in this case repeating the request works.
// // Error is:
// // request to https://graph.microsoft.com/v1.0/drive/special/approot failed, reason: getaddrinfo ENOTFOUND graph.microsoft.com graph.microsoft.com:443
// if (error.code == 'ENOTFOUND') return true;
// network timeout at: https://public-ch3302...859f9b0e3ab.md
if (error.message && error.message.indexOf('network timeout') === 0) return true;
// // network timeout at: https://public-ch3302...859f9b0e3ab.md
// if (error.message && error.message.indexOf('network timeout') === 0) return true;
// name: 'FetchError',
// message: 'request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443',
// type: 'system',
// errno: 'EAI_AGAIN',
// code: 'EAI_AGAIN' } } reason: { FetchError: request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443
//
// It's a Microsoft error: "A temporary failure in name resolution occurred."
if (error.code == 'EAI_AGAIN') return true;
// // name: 'FetchError',
// // message: 'request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443',
// // type: 'system',
// // errno: 'EAI_AGAIN',
// // code: 'EAI_AGAIN' } } reason: { FetchError: request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443
// //
// // It's a Microsoft error: "A temporary failure in name resolution occurred."
// if (error.code == 'EAI_AGAIN') return true;
return false;
}
// // request to https://public-...8fd8bc6bb68e9c4d17a.md failed, reason: connect ETIMEDOUT 204.79.197.213:443
// // Code: ETIMEDOUT
// if (error.code === 'ETIMEDOUT') return true;
// return false;
// }
function shimInit() {
shim.fs = fs;
@@ -110,31 +117,35 @@ function shimInit() {
}
shim.fetch = async function(url, options = null) {
if (!options) options = {};
if (!options.timeout) options.timeout = 1000 * 120; // ms
if (!('maxRetry' in options)) options.maxRetry = 5;
return shim.fetchWithRetry(() => {
return nodeFetch(url, options)
}, options);
let retryCount = 0;
while (true) {
try {
const response = await nodeFetch(url, options);
return response;
} catch (error) {
if (fetchRequestCanBeRetried(error)) {
retryCount++;
if (retryCount > options.maxRetry) throw error;
await time.sleep(retryCount * 3);
} else {
throw error;
}
}
}
// if (!options) options = {};
// if (!options.timeout) options.timeout = 1000 * 120; // ms
// if (!('maxRetry' in options)) options.maxRetry = 5;
// let retryCount = 0;
// while (true) {
// try {
// const response = await nodeFetch(url, options);
// return response;
// } catch (error) {
// if (fetchRequestCanBeRetried(error)) {
// retryCount++;
// if (retryCount > options.maxRetry) throw error;
// await time.sleep(retryCount * 3);
// } else {
// throw error;
// }
// }
// }
}
shim.fetchBlob = async function(url, options) {
if (!options || !options.path) throw new Error('fetchBlob: target file path is missing');
if (!options.method) options.method = 'GET';
if (!('maxRetry' in options)) options.maxRetry = 5;
//if (!('maxRetry' in options)) options.maxRetry = 5;
const urlParse = require('url').parse;
@@ -192,21 +203,23 @@ function shimInit() {
});
};
let retryCount = 0;
while (true) {
try {
const response = await doFetchOperation();
return response;
} catch (error) {
if (fetchRequestCanBeRetried(error)) {
retryCount++;
if (retryCount > options.maxRetry) throw error;
await time.sleep(retryCount * 3);
} else {
throw error;
}
}
}
return shim.fetchWithRetry(doFetchOperation, options);
// let retryCount = 0;
// while (true) {
// try {
// const response = await doFetchOperation();
// return response;
// } catch (error) {
// if (fetchRequestCanBeRetried(error)) {
// retryCount++;
// if (retryCount > options.maxRetry) throw error;
// await time.sleep(retryCount * 3);
// } else {
// throw error;
// }
// }
// }
}
}

37
ReactNativeClient/lib/shim-init-react.js
View File

@@ -9,6 +9,32 @@ function shimInit() {
shim.setInterval = PoorManIntervals.setInterval;
shim.clearInterval = PoorManIntervals.clearInterval;
shim.fetch = async function(url, options = null) {
return shim.fetchWithRetry(() => {
return shim.nativeFetch_(url, options)
}, options);
// if (!options) options = {};
// if (!options.timeout) options.timeout = 1000 * 120; // ms
// if (!('maxRetry' in options)) options.maxRetry = 5;
// let retryCount = 0;
// while (true) {
// try {
// const response = await nodeFetch(url, options);
// return response;
// } catch (error) {
// if (fetchRequestCanBeRetried(error)) {
// retryCount++;
// if (retryCount > options.maxRetry) throw error;
// await time.sleep(retryCount * 3);
// } else {
// throw error;
// }
// }
// }
}
shim.fetchBlob = async function(url, options) {
if (!options || !options.path) throw new Error('fetchBlob: target file path is missing');
@@ -21,10 +47,17 @@ function shimInit() {
delete options.path;
try {
let response = await RNFetchBlob.config({
const doFetchBlob = () => {
return RNFetchBlob.config({
path: localFilePath
}).fetch(method, url, headers);
}
try {
const response = await shim.fetchWithRetry(doFetchBlob, options);
// let response = await RNFetchBlob.config({
// path: localFilePath
// }).fetch(method, url, headers);
// Returns an object that's roughtly compatible with a standard Response object
let output = {

65
ReactNativeClient/lib/shim.js
View File

@@ -30,7 +30,70 @@ shim.isElectron = () => {
return false;
}
shim.fetch = typeof fetch !== 'undefined' ? fetch : null;
// Node requests can go wrong is so many different ways and with so
// many different error messages... This handler inspects the error
// and decides whether the request can safely be repeated or not.
function fetchRequestCanBeRetried(error) {
if (!error) return false;
// Unfortunately the error 'Network request failed' doesn't have a type
// or error code, so hopefully that message won't change and is not localized
if (error.message == 'Network request failed') return true;
// request to https://public-ch3302....1fab24cb1bd5f.md failed, reason: socket hang up"
if (error.code == 'ECONNRESET') return true;
// OneDrive (or Node?) sometimes sends back a "not found" error for resources
// that definitely exist and in this case repeating the request works.
// Error is:
// request to https://graph.microsoft.com/v1.0/drive/special/approot failed, reason: getaddrinfo ENOTFOUND graph.microsoft.com graph.microsoft.com:443
if (error.code == 'ENOTFOUND') return true;
// network timeout at: https://public-ch3302...859f9b0e3ab.md
if (error.message && error.message.indexOf('network timeout') === 0) return true;
// name: 'FetchError',
// message: 'request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443',
// type: 'system',
// errno: 'EAI_AGAIN',
// code: 'EAI_AGAIN' } } reason: { FetchError: request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443
//
// It's a Microsoft error: "A temporary failure in name resolution occurred."
if (error.code == 'EAI_AGAIN') return true;
// request to https://public-...8fd8bc6bb68e9c4d17a.md failed, reason: connect ETIMEDOUT 204.79.197.213:443
// Code: ETIMEDOUT
if (error.code === 'ETIMEDOUT') return true;
return false;
}
shim.fetchWithRetry = async function(fetchFn, options = null) {
const { time } = require('lib/time-utils.js');
if (!options) options = {};
if (!options.timeout) options.timeout = 1000 * 120; // ms
if (!('maxRetry' in options)) options.maxRetry = 5;
let retryCount = 0;
while (true) {
try {
const response = await fetchFn();
return response;
} catch (error) {
if (fetchRequestCanBeRetried(error)) {
retryCount++;
if (retryCount > options.maxRetry) throw error;
await time.sleep(retryCount * 3);
} else {
throw error;
}
}
}
}
shim.nativeFetch_ = typeof fetch !== 'undefined' ? fetch : null;
shim.fetch = () => { throw new Error('Not implemented'); }
shim.FormData = typeof FormData !== 'undefined' ? FormData : null;
shim.fs = null;
shim.FileApiDriverLocal = null;