self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-11-27 08:21:03 +02:00
Code Issues Releases Activity

Improved fetch with retry on all three platforms

Browse Source
This commit is contained in:
Laurent Cozic 2017-11-12 22:52:54 +00:00
parent 71a97bd45b
commit 2cf5234e76
10 changed files with 223 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ElectronClient/app/gui/OneDriveLoginScreen.jsx
View File

@ -101,7 +101,9 @@ class OneDriveLoginScreenComponent extends React.Component {
}
const mapStateToProps = (state) => {
return {};
return {
theme: state.settings.theme,
};
};
const OneDriveLoginScreen = connect(mapStateToProps)(OneDriveLoginScreenComponent);

69
ElectronClient/app/package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "joplin-desktop",
"version": "0.0.1",
"version": "0.10.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -104,7 +104,7 @@
"integrity": "sha512-NnSOmMEYtVR2JVMIGTzynRkkaxtiq1xnFBcdQD/DnNCYPoEPsVJhM98BDyaoNOQIi7p4okdi3E27eN7GQbsUug==",
"dev": true,
"requires": {
"color-convert": "1.9.0"
"color-convert": "1.9.1"
}
},
"anymatch": {
@ -124,17 +124,16 @@
"integrity": "sha1-ZBqlXft9am8KgUHEucCqULbCTdU="
},
"app-package-builder": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/app-package-builder/-/app-package-builder-1.3.3.tgz",
"integrity": "sha512-aBcCNFQ4GDQ0AOYvTmd/g2L2zoV+RnK6TQnNm3IoakdonQ4iqiyOZFGc6i53SYEnB5JRcZEdpO7RjXeO8Nr9sg==",
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/app-package-builder/-/app-package-builder-1.5.0.tgz",
"integrity": "sha512-CmmmXq543Ru7TEcAQShmmI9Z0NgKZ54KOR3BGM+LTEw/v2VjgCQ5qrkkoCA+0LaYgJfPFg/40PGMHnDaAZ6g4w==",
"dev": true,
"requires": {
"bluebird-lst": "1.0.5",
"builder-util": "3.2.0",
"builder-util-runtime": "2.5.0",
"builder-util": "3.2.2",
"builder-util-runtime": "3.2.0",
"fs-extra-p": "4.4.4",
"int64-buffer": "0.1.9",
"js-yaml": "3.10.0",
"rabin-bindings": "1.7.3"
}
},
@ -811,14 +810,14 @@
}
},
"builder-util": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/builder-util/-/builder-util-3.2.0.tgz",
"integrity": "sha512-TWvixCYS4fyZgrn4tSeKfsO1j13zX8MGWRgLdsWP0op4SD0Lp4NJJVJjvwBzHIfOCcxpQsszgbcSMBE2lrMVjA==",
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/builder-util/-/builder-util-3.2.2.tgz",
"integrity": "sha512-4t/EtpYYp5wNaNlI4cgKbHEY/oNtnKtuEn201DNcJlgLkoV8RrwSOo6RJbDOKY+RTf0IVmbEZ2ZHbyy2/KGKRQ==",
"dev": true,
"requires": {
"7zip-bin": "2.2.7",
"bluebird-lst": "1.0.5",
"builder-util-runtime": "2.5.0",
"builder-util-runtime": "3.2.0",
"chalk": "2.3.0",
"debug": "3.1.0",
"fs-extra-p": "4.4.4",
@ -835,9 +834,9 @@
}
},
"builder-util-runtime": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-2.5.0.tgz",
"integrity": "sha512-1D/jB5FDc8RP8UxPvUo7UwcChNiDviJks57LUkWeGY8TLCoQLljDUamOBAUYXeZ4HnAqwuTsVgWgx5jxmayl5Q==",
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/builder-util-runtime/-/builder-util-runtime-3.2.0.tgz",
"integrity": "sha512-VRvyyLiZZSBjcUTqEsHlBJSK0s6uVQChO7kbmVeU6QmSJ7TtsotNQELO6lbahwZMAQ4Z/haCKhlLBDdhW+3aqA==",
"dev": true,
"requires": {
"bluebird-lst": "1.0.5",
@ -1005,9 +1004,9 @@
}
},
"color-convert": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.0.tgz",
"integrity": "sha1-Gsz5fdc5uYO/mU1W/sj5WFNkG3o=",
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz",
"integrity": "sha512-mjGanIiwQJskCC18rPR6OmrZ6fm2Lc7PeGFYwCmy5J34wC6F1PzdGL6xeMfmgicfYcNLGuVFA3WzXtIDCQSZxQ==",
"dev": true,
"requires": {
"color-name": "1.1.3"
@ -1251,13 +1250,13 @@
"integrity": "sha1-R/31ZzSKF+wl/L8LnkRjSKdvn7U="
},
"dmg-builder": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/dmg-builder/-/dmg-builder-2.1.5.tgz",
"integrity": "sha512-KbldWIhaP3XSI7sYPGrBcNwSPxvB2ImIMSqz1tKcIqVZ9P+9UvYtdIJvM6Og6DSyAQlwX37E0soKwD/eddGFUA==",
"version": "2.1.6",
"resolved": "https://registry.npmjs.org/dmg-builder/-/dmg-builder-2.1.6.tgz",
"integrity": "sha512-5gYVdJJPnATbnMAv/ufMd2qqbb8wWLyl77am1EKUL+5ZZjCkujps1inHcq8+s+G/iJJqGn1ko4I7RBvhd2AV/w==",
"dev": true,
"requires": {
"bluebird-lst": "1.0.5",
"builder-util": "3.2.0",
"builder-util": "3.2.2",
"debug": "3.1.0",
"fs-extra-p": "4.4.4",
"iconv-lite": "0.4.19",
@ -1389,26 +1388,26 @@
}
},
"electron-builder": {
"version": "19.43.4",
"resolved": "https://registry.npmjs.org/electron-builder/-/electron-builder-19.43.4.tgz",
"integrity": "sha512-rGpJwOAHgIEZ6Ck0ogxViQ0/NtAS7hzlh72xY31QazCjgSwKLAkWHGHP+T4U12EzVqFOmI3pD1i6/eBeAEUMuw==",
"version": "19.45.4",
"resolved": "https://registry.npmjs.org/electron-builder/-/electron-builder-19.45.4.tgz",
"integrity": "sha512-WuhPBFFU/rdx9ehwS8tOnZfvzTuip7UzZCRZVQy0Bimypv9ZQ65WuCAx8MSy4gLDQ1et63Vrl+fUvtXGQIZI8g==",
"dev": true,
"requires": {
"7zip-bin": "2.2.7",
"app-package-builder": "1.3.3",
"app-package-builder": "1.5.0",
"asar-integrity": "0.2.3",
"async-exit-hook": "2.0.1",
"bluebird-lst": "1.0.5",
"builder-util": "3.2.0",
"builder-util-runtime": "2.5.0",
"builder-util": "3.2.2",
"builder-util-runtime": "3.2.0",
"chalk": "2.3.0",
"chromium-pickle-js": "0.2.0",
"debug": "3.1.0",
"dmg-builder": "2.1.5",
"dmg-builder": "2.1.6",
"ejs": "2.5.7",
"electron-download-tf": "4.3.4",
"electron-osx-sign": "0.4.7",
"electron-publish": "19.43.0",
"electron-publish": "19.45.0",
"fs-extra-p": "4.4.4",
"hosted-git-info": "2.5.0",
"is-ci": "1.0.10",
@ -1493,14 +1492,14 @@
}
},
"electron-publish": {
"version": "19.43.0",
"resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-19.43.0.tgz",
"integrity": "sha512-gsFmUsdzfB2EXtMHJD+jRJgvREgUUJasGbZsoaRiJQr5kJZruUQOkIq+NMRsN9o3eQvu0fhxdotDhH4W+U1ebA==",
"version": "19.45.0",
"resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-19.45.0.tgz",
"integrity": "sha512-hubOvL3bvgDoITy0hgh+FSbC2Nx1OYDeE4ipzSnEEa6xzjS9svROxUiyyPncbL9O1E+VreJTDyNWaYsYaUg8jw==",
"dev": true,
"requires": {
"bluebird-lst": "1.0.5",
"builder-util": "3.2.0",
"builder-util-runtime": "2.5.0",
"builder-util": "3.2.2",
"builder-util-runtime": "3.2.0",
"chalk": "2.3.0",
"fs-extra-p": "4.4.4",
"mime": "2.0.3"

9
ElectronClient/app/package.json
View File

@ -5,7 +5,9 @@
"main": "main.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"postinstall": "electron-builder install-app-deps"
"postinstall": "electron-builder install-app-deps",
"pack": "electron-builder --dir",
"dist": "electron-builder"
},
"repository": {
"type": "git",
@ -16,12 +18,15 @@
"bugs": {
"url": "https://github.com/laurent22/joplin/issues"
},
"build": {
"appId": "net.cozic.joplin-desktop"
},
"homepage": "https://github.com/laurent22/joplin#readme",
"devDependencies": {
"babel-cli": "^6.26.0",
"babel-preset-react": "^6.24.1",
"electron": "^1.7.9",
"electron-builder": "^19.43.4",
"electron-builder": "^19.45.4",
"electron-rebuild": "^1.6.0"
},
"dependencies": {

6
ElectronClient/run-prod.sh Normal file
View File

@ -0,0 +1,6 @@
#!/bin/bash
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$ROOT_DIR"
./build.sh || exit 1
cd "$ROOT_DIR/app"
./node_modules/.bin/electron . --log-level debug "$@"

4
ReactNativeClient/lib/net-utils.js
View File

@ -1,7 +1,5 @@
const { shim } = require('lib/shim.js');
const tcpPortUsed = require('tcp-port-used');
const netUtils = {};
netUtils.ip = async () => {
@ -15,6 +13,8 @@ netUtils.ip = async () => {
}
netUtils.findAvailablePort = async (possiblePorts, extraRandomPortsToTry = 20) => {
const tcpPortUsed = require('tcp-port-used');
for (let i = 0; i < extraRandomPortsToTry; i++) {
possiblePorts.push(Math.floor(8000 + Math.random() * 2000));
}

24
ReactNativeClient/lib/onedrive-api.js
View File

@ -177,32 +177,8 @@ class OneDriveApi {
response = await shim.fetchBlob(url, options);
}
} catch (error) {
// let canRetry = true;
// if (error.message == 'Network request failed') {
// // Unfortunately the error 'Network request failed' doesn't have a type
// // or error code, so hopefully that message won't change and is not localized
// } else if (error.code == 'ECONNRESET') {
// // request to https://public-ch3302....1fab24cb1bd5f.md failed, reason: socket hang up"
// } else if (error.code == 'ENOTFOUND') {
// // OneDrive (or Node?) sometimes sends back a "not found" error for resources
// // that definitely exist and in this case repeating the request works.
// // Error is:
// // request to https://graph.microsoft.com/v1.0/drive/special/approot failed, reason: getaddrinfo ENOTFOUND graph.microsoft.com graph.microsoft.com:443
// } else if (error.message.indexOf('network timeout') === 0) {
// // network timeout at: https://public-ch3302...859f9b0e3ab.md
// } else {
// canRetry = false;
// }
// if (canRetry) {
// this.logger().info('Got error code ' + error.code + ': ' + error.message + ' - retrying (' + i + ')...');
// await time.sleep((i + 1) * 3);
// continue;
// } else {
this.logger().error('Got unhandled error:', error ? error.code : '', error ? error.message : '', error);
throw error;
//}
}
if (!response.ok) {

129
ReactNativeClient/lib/shim-init-node.js
View File

@ -5,36 +5,43 @@ const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
const { time } = require('lib/time-utils.js');
const { setLocale, defaultLocale, closestSupportedLocale } = require('lib/locale.js');
function fetchRequestCanBeRetried(error) {
if (!error) return false;
// // Node requests can go wrong is so many different ways and with so
// // many different error messages... This handler inspects the error
// // and decides whether the request can safely be repeated or not.
// function fetchRequestCanBeRetried(error) {
// if (!error) return false;
// Unfortunately the error 'Network request failed' doesn't have a type
// or error code, so hopefully that message won't change and is not localized
if (error.message == 'Network request failed') return true;
// // Unfortunately the error 'Network request failed' doesn't have a type
// // or error code, so hopefully that message won't change and is not localized
// if (error.message == 'Network request failed') return true;
// request to https://public-ch3302....1fab24cb1bd5f.md failed, reason: socket hang up"
if (error.code == 'ECONNRESET') return true;
// // request to https://public-ch3302....1fab24cb1bd5f.md failed, reason: socket hang up"
// if (error.code == 'ECONNRESET') return true;
// OneDrive (or Node?) sometimes sends back a "not found" error for resources
// that definitely exist and in this case repeating the request works.
// Error is:
// request to https://graph.microsoft.com/v1.0/drive/special/approot failed, reason: getaddrinfo ENOTFOUND graph.microsoft.com graph.microsoft.com:443
if (error.code == 'ENOTFOUND') return true;
// // OneDrive (or Node?) sometimes sends back a "not found" error for resources
// // that definitely exist and in this case repeating the request works.
// // Error is:
// // request to https://graph.microsoft.com/v1.0/drive/special/approot failed, reason: getaddrinfo ENOTFOUND graph.microsoft.com graph.microsoft.com:443
// if (error.code == 'ENOTFOUND') return true;
// network timeout at: https://public-ch3302...859f9b0e3ab.md
if (error.message && error.message.indexOf('network timeout') === 0) return true;
// // network timeout at: https://public-ch3302...859f9b0e3ab.md
// if (error.message && error.message.indexOf('network timeout') === 0) return true;
// name: 'FetchError',
// message: 'request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443',
// type: 'system',
// errno: 'EAI_AGAIN',
// code: 'EAI_AGAIN' } } reason: { FetchError: request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443
//
// It's a Microsoft error: "A temporary failure in name resolution occurred."
if (error.code == 'EAI_AGAIN') return true;
// // name: 'FetchError',
// // message: 'request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443',
// // type: 'system',
// // errno: 'EAI_AGAIN',
// // code: 'EAI_AGAIN' } } reason: { FetchError: request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443
// //
// // It's a Microsoft error: "A temporary failure in name resolution occurred."
// if (error.code == 'EAI_AGAIN') return true;
return false;
}
// // request to https://public-...8fd8bc6bb68e9c4d17a.md failed, reason: connect ETIMEDOUT 204.79.197.213:443
// // Code: ETIMEDOUT
// if (error.code === 'ETIMEDOUT') return true;
// return false;
// }
function shimInit() {
shim.fs = fs;
@ -110,31 +117,35 @@ function shimInit() {
}
shim.fetch = async function(url, options = null) {
if (!options) options = {};
if (!options.timeout) options.timeout = 1000 * 120; // ms
if (!('maxRetry' in options)) options.maxRetry = 5;
return shim.fetchWithRetry(() => {
return nodeFetch(url, options)
}, options);
let retryCount = 0;
while (true) {
try {
const response = await nodeFetch(url, options);
return response;
} catch (error) {
if (fetchRequestCanBeRetried(error)) {
retryCount++;
if (retryCount > options.maxRetry) throw error;
await time.sleep(retryCount * 3);
} else {
throw error;
}
}
}
// if (!options) options = {};
// if (!options.timeout) options.timeout = 1000 * 120; // ms
// if (!('maxRetry' in options)) options.maxRetry = 5;
// let retryCount = 0;
// while (true) {
// try {
// const response = await nodeFetch(url, options);
// return response;
// } catch (error) {
// if (fetchRequestCanBeRetried(error)) {
// retryCount++;
// if (retryCount > options.maxRetry) throw error;
// await time.sleep(retryCount * 3);
// } else {
// throw error;
// }
// }
// }
}
shim.fetchBlob = async function(url, options) {
if (!options || !options.path) throw new Error('fetchBlob: target file path is missing');
if (!options.method) options.method = 'GET';
if (!('maxRetry' in options)) options.maxRetry = 5;
//if (!('maxRetry' in options)) options.maxRetry = 5;
const urlParse = require('url').parse;
@ -192,21 +203,23 @@ function shimInit() {
});
};
let retryCount = 0;
while (true) {
try {
const response = await doFetchOperation();
return response;
} catch (error) {
if (fetchRequestCanBeRetried(error)) {
retryCount++;
if (retryCount > options.maxRetry) throw error;
await time.sleep(retryCount * 3);
} else {
throw error;
}
}
}
return shim.fetchWithRetry(doFetchOperation, options);
// let retryCount = 0;
// while (true) {
// try {
// const response = await doFetchOperation();
// return response;
// } catch (error) {
// if (fetchRequestCanBeRetried(error)) {
// retryCount++;
// if (retryCount > options.maxRetry) throw error;
// await time.sleep(retryCount * 3);
// } else {
// throw error;
// }
// }
// }
}
}

37
ReactNativeClient/lib/shim-init-react.js
View File

@ -9,6 +9,32 @@ function shimInit() {
shim.setInterval = PoorManIntervals.setInterval;
shim.clearInterval = PoorManIntervals.clearInterval;
shim.fetch = async function(url, options = null) {
return shim.fetchWithRetry(() => {
return shim.nativeFetch_(url, options)
}, options);
// if (!options) options = {};
// if (!options.timeout) options.timeout = 1000 * 120; // ms
// if (!('maxRetry' in options)) options.maxRetry = 5;
// let retryCount = 0;
// while (true) {
// try {
// const response = await nodeFetch(url, options);
// return response;
// } catch (error) {
// if (fetchRequestCanBeRetried(error)) {
// retryCount++;
// if (retryCount > options.maxRetry) throw error;
// await time.sleep(retryCount * 3);
// } else {
// throw error;
// }
// }
// }
}
shim.fetchBlob = async function(url, options) {
if (!options || !options.path) throw new Error('fetchBlob: target file path is missing');
@ -21,10 +47,17 @@ function shimInit() {
delete options.path;
try {
let response = await RNFetchBlob.config({
const doFetchBlob = () => {
return RNFetchBlob.config({
path: localFilePath
}).fetch(method, url, headers);
}
try {
const response = await shim.fetchWithRetry(doFetchBlob, options);
// let response = await RNFetchBlob.config({
// path: localFilePath
// }).fetch(method, url, headers);
// Returns an object that's roughtly compatible with a standard Response object
let output = {

65
ReactNativeClient/lib/shim.js
View File

@ -30,7 +30,70 @@ shim.isElectron = () => {
return false;
}
shim.fetch = typeof fetch !== 'undefined' ? fetch : null;
// Node requests can go wrong is so many different ways and with so
// many different error messages... This handler inspects the error
// and decides whether the request can safely be repeated or not.
function fetchRequestCanBeRetried(error) {
if (!error) return false;
// Unfortunately the error 'Network request failed' doesn't have a type
// or error code, so hopefully that message won't change and is not localized
if (error.message == 'Network request failed') return true;
// request to https://public-ch3302....1fab24cb1bd5f.md failed, reason: socket hang up"
if (error.code == 'ECONNRESET') return true;
// OneDrive (or Node?) sometimes sends back a "not found" error for resources
// that definitely exist and in this case repeating the request works.
// Error is:
// request to https://graph.microsoft.com/v1.0/drive/special/approot failed, reason: getaddrinfo ENOTFOUND graph.microsoft.com graph.microsoft.com:443
if (error.code == 'ENOTFOUND') return true;
// network timeout at: https://public-ch3302...859f9b0e3ab.md
if (error.message && error.message.indexOf('network timeout') === 0) return true;
// name: 'FetchError',
// message: 'request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443',
// type: 'system',
// errno: 'EAI_AGAIN',
// code: 'EAI_AGAIN' } } reason: { FetchError: request to https://api.ipify.org/?format=json failed, reason: getaddrinfo EAI_AGAIN api.ipify.org:443
//
// It's a Microsoft error: "A temporary failure in name resolution occurred."
if (error.code == 'EAI_AGAIN') return true;
// request to https://public-...8fd8bc6bb68e9c4d17a.md failed, reason: connect ETIMEDOUT 204.79.197.213:443
// Code: ETIMEDOUT
if (error.code === 'ETIMEDOUT') return true;
return false;
}
shim.fetchWithRetry = async function(fetchFn, options = null) {
const { time } = require('lib/time-utils.js');
if (!options) options = {};
if (!options.timeout) options.timeout = 1000 * 120; // ms
if (!('maxRetry' in options)) options.maxRetry = 5;
let retryCount = 0;
while (true) {
try {
const response = await fetchFn();
return response;
} catch (error) {
if (fetchRequestCanBeRetried(error)) {
retryCount++;
if (retryCount > options.maxRetry) throw error;
await time.sleep(retryCount * 3);
} else {
throw error;
}
}
}
}
shim.nativeFetch_ = typeof fetch !== 'undefined' ? fetch : null;
shim.fetch = () => { throw new Error('Not implemented'); }
shim.FormData = typeof FormData !== 'undefined' ? FormData : null;
shim.fs = null;
shim.FileApiDriverLocal = null;

1
joplin.sublime-project
View File

@ -35,6 +35,7 @@
"CliClient/tests/fuzzing",
"CliClient/tests/src",
"CliClient/tests/sync",
"ElectronClient/dist",
"ElectronClient/build",
"ElectronClient/app/lib",
"ElectronClient/app/locale",