mirror of
https://github.com/laurent22/joplin.git
synced 2024-12-24 10:27:10 +02:00
Revert: All: Resolves #4810, Resolves #4610: Fix AWS S3 sync error and upgrade framework to v3 (#5212)
Revert commit d2e2866995
Ref: https://github.com/laurent22/joplin/issues/5287
This commit is contained in:
Laurent Cozic
parent
a3b37ca59d
commit
6927335330
5243
packages/app-cli/package-lock.json
generated
5243
packages/app-cli/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -42,6 +42,7 @@
|
||||
"dependencies": {
|
||||
"@joplin/lib": "~2.2",
|
||||
"@joplin/renderer": "~2.2",
|
||||
"aws-sdk": "^2.588.0",
|
||||
"chalk": "^4.1.0",
|
||||
"compare-version": "^0.1.2",
|
||||
"fs-extra": "^5.0.0",
|
||||
|
||||
@ -6,10 +6,6 @@
|
||||
|
||||
// So there's basically still a one way flux: React => SQLite => Redux => React
|
||||
|
||||
// For aws-sdk-js-v3
|
||||
import 'react-native-get-random-values';
|
||||
import 'react-native-url-polyfill/auto';
|
||||
|
||||
import { LogBox, AppRegistry } from 'react-native';
|
||||
const Root = require('./root').default;
|
||||
|
||||
|
||||
@ -221,8 +221,6 @@ PODS:
|
||||
- React-Core
|
||||
- react-native-geolocation (2.0.2):
|
||||
- React
|
||||
- react-native-get-random-values (1.7.0):
|
||||
- React-Core
|
||||
- react-native-image-picker (2.3.4):
|
||||
- React-Core
|
||||
- react-native-image-resizer (1.3.0):
|
||||
@ -347,7 +345,6 @@ DEPENDENCIES:
|
||||
- react-native-camera (from `../node_modules/react-native-camera`)
|
||||
- react-native-document-picker (from `../node_modules/react-native-document-picker`)
|
||||
- "react-native-geolocation (from `../node_modules/@react-native-community/geolocation`)"
|
||||
- react-native-get-random-values (from `../node_modules/react-native-get-random-values`)
|
||||
- react-native-image-picker (from `../node_modules/react-native-image-picker`)
|
||||
- react-native-image-resizer (from `../node_modules/react-native-image-resizer`)
|
||||
- "react-native-netinfo (from `../node_modules/@react-native-community/netinfo`)"
|
||||
@ -426,8 +423,6 @@ EXTERNAL SOURCES:
|
||||
:path: "../node_modules/react-native-document-picker"
|
||||
react-native-geolocation:
|
||||
:path: "../node_modules/@react-native-community/geolocation"
|
||||
react-native-get-random-values:
|
||||
:path: "../node_modules/react-native-get-random-values"
|
||||
react-native-image-picker:
|
||||
:path: "../node_modules/react-native-image-picker"
|
||||
react-native-image-resizer:
|
||||
@ -512,7 +507,6 @@ SPEC CHECKSUMS:
|
||||
react-native-camera: 5c1fbfecf63b802b8ca4a71c60d30a71550fb348
|
||||
react-native-document-picker: b3e78a8f7fef98b5cb069f20fc35797d55e68e28
|
||||
react-native-geolocation: cbd9d6bd06bac411eed2671810f454d4908484a8
|
||||
react-native-get-random-values: 237bffb1c7e05fb142092681531810a29ba53015
|
||||
react-native-image-picker: 32d1ad2c0024ca36161ae0d5c2117e2d6c441f11
|
||||
react-native-image-resizer: b53bf95ad880100e20262687e41f76fdbc9df255
|
||||
react-native-netinfo: 34f4d7a42f49157f3b45c14217d256bce7dc9682
|
||||
|
||||
57
packages/app-mobile/package-lock.json
generated
57
packages/app-mobile/package-lock.json
generated
@ -3636,11 +3636,6 @@
|
||||
"time-stamp": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"fast-base64-decode": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-base64-decode/-/fast-base64-decode-1.0.0.tgz",
|
||||
"integrity": "sha512-qwaScUgUGBYeDNRnbc/KyllVU88Jk1pRHPStuF/lO7B0/RTRLj7U0lkdTAutlBblY08rwZDff6tNU9cjv6j//Q=="
|
||||
},
|
||||
"fb-watchman": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.1.tgz",
|
||||
@ -7073,11 +7068,6 @@
|
||||
"escape-goat": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"querystring": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
|
||||
"integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
|
||||
},
|
||||
"range-parser": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
||||
@ -7352,14 +7342,6 @@
|
||||
"utf8": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"react-native-get-random-values": {
|
||||
"version": "1.7.0",
|
||||
"resolved": "https://registry.npmjs.org/react-native-get-random-values/-/react-native-get-random-values-1.7.0.tgz",
|
||||
"integrity": "sha512-zDhmpWUekGRFb9I+MQkxllHcqXN9HBSsgPwBQfrZ1KZYpzDspWLZ6/yLMMZrtq4pVqNR7C7N96L3SuLpXv1nhQ==",
|
||||
"requires": {
|
||||
"fast-base64-decode": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"react-native-image-picker": {
|
||||
"version": "2.3.4",
|
||||
"resolved": "https://registry.npmjs.org/react-native-image-picker/-/react-native-image-picker-2.3.4.tgz",
|
||||
@ -7423,14 +7405,6 @@
|
||||
"resolved": "https://registry.npmjs.org/react-native-sqlite-storage/-/react-native-sqlite-storage-5.0.0.tgz",
|
||||
"integrity": "sha512-c1Joq3/tO1nmIcP8SkRZNolPSbfvY8uZg5lXse0TmjIPC0qHVbk96IMvWGyly1TmYCIpxpuDRc0/xCffDbYIvg=="
|
||||
},
|
||||
"react-native-url-polyfill": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/react-native-url-polyfill/-/react-native-url-polyfill-1.3.0.tgz",
|
||||
"integrity": "sha512-w9JfSkvpqqlix9UjDvJjm1EjSt652zVQ6iwCIj1cVVkwXf4jQhQgTNXY6EVTwuAmUjg6BC6k9RHCBynoLFo3IQ==",
|
||||
"requires": {
|
||||
"whatwg-url-without-unicode": "8.0.0-3"
|
||||
}
|
||||
},
|
||||
"react-native-vector-icons": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/react-native-vector-icons/-/react-native-vector-icons-7.1.0.tgz",
|
||||
@ -9092,22 +9066,6 @@
|
||||
"resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz",
|
||||
"integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI="
|
||||
},
|
||||
"url": {
|
||||
"version": "0.11.0",
|
||||
"resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
|
||||
"integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
|
||||
"requires": {
|
||||
"punycode": "1.3.2",
|
||||
"querystring": "0.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"punycode": {
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
|
||||
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
|
||||
}
|
||||
}
|
||||
},
|
||||
"url-parse-lax": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz",
|
||||
@ -9274,26 +9232,11 @@
|
||||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"webidl-conversions": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz",
|
||||
"integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA=="
|
||||
},
|
||||
"whatwg-fetch": {
|
||||
"version": "3.4.1",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.4.1.tgz",
|
||||
"integrity": "sha512-sofZVzE1wKwO+EYPbWfiwzaKovWiZXf4coEzjGP9b2GBVgQRLQUZ2QcuPpQExGDAW5GItpEm6Tl4OU5mywnAoQ=="
|
||||
},
|
||||
"whatwg-url-without-unicode": {
|
||||
"version": "8.0.0-3",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url-without-unicode/-/whatwg-url-without-unicode-8.0.0-3.tgz",
|
||||
"integrity": "sha512-HoKuzZrUlgpz35YO27XgD28uh/WJH4B0+3ttFqRo//lmq+9T/mIOJ6kqmINI9HpUpz1imRC/nR/lxKpJiv0uig==",
|
||||
"requires": {
|
||||
"buffer": "^5.4.3",
|
||||
"punycode": "^2.1.1",
|
||||
"webidl-conversions": "^5.0.0"
|
||||
}
|
||||
},
|
||||
"which": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
||||
|
||||
@ -39,7 +39,6 @@
|
||||
"react-native-dropdownalert": "^3.1.2",
|
||||
"react-native-file-viewer": "^2.1.4",
|
||||
"react-native-fs": "^2.16.6",
|
||||
"react-native-get-random-values": "^1.7.0",
|
||||
"react-native-image-picker": "^2.3.4",
|
||||
"react-native-image-resizer": "^1.3.0",
|
||||
"react-native-modal-datetime-picker": "^9.0.0",
|
||||
@ -50,7 +49,6 @@
|
||||
"react-native-share": "^5.1.5",
|
||||
"react-native-side-menu": "^1.1.3",
|
||||
"react-native-sqlite-storage": "^5.0.0",
|
||||
"react-native-url-polyfill": "^1.3.0",
|
||||
"react-native-vector-icons": "^7.1.0",
|
||||
"react-native-version-info": "^1.1.0",
|
||||
"react-native-webview": "^10.9.2",
|
||||
@ -61,7 +59,6 @@
|
||||
"stream-browserify": "^3.0.0",
|
||||
"string-natural-compare": "^2.0.2",
|
||||
"timers": "^0.1.1",
|
||||
"url": "^0.11.0",
|
||||
"valid-url": "^1.0.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import FsDriverBase from '@joplin/lib/fs-driver-base';
|
||||
const RNFetchBlob = require('rn-fetch-blob').default;
|
||||
const RNFS = require('react-native-fs');
|
||||
const { Writable } = require('stream-browserify');
|
||||
const { Buffer } = require('buffer');
|
||||
|
||||
export default class FsDriverRN extends FsDriverBase {
|
||||
public appendFileSync() {
|
||||
@ -24,6 +26,27 @@ export default class FsDriverRN extends FsDriverBase {
|
||||
return await this.unlink(path);
|
||||
}
|
||||
|
||||
public writeBinaryFile(path: string, content: any) {
|
||||
const buffer = Buffer.from(content);
|
||||
return RNFetchBlob.fs.writeStream(path, 'base64').then((stream: any) => {
|
||||
const fileStream = new Writable({
|
||||
write(chunk: any, _encoding: any, callback: Function) {
|
||||
this.stream.write(chunk.toString('base64'));
|
||||
callback();
|
||||
},
|
||||
final(callback: Function) {
|
||||
this.stream.close();
|
||||
callback();
|
||||
},
|
||||
});
|
||||
// using options.construct is not implemented in readable-stream so lets
|
||||
// pass the stream from RNFetchBlob to the Writable instance here
|
||||
fileStream.stream = stream;
|
||||
fileStream.write(buffer);
|
||||
fileStream.end();
|
||||
});
|
||||
}
|
||||
|
||||
// Returns a format compatible with Node.js format
|
||||
private rnfsStatToStd_(stat: any, path: string) {
|
||||
return {
|
||||
|
||||
@ -4,7 +4,7 @@ const Setting = require('./models/Setting').default;
|
||||
const { FileApi } = require('./file-api.js');
|
||||
const Synchronizer = require('./Synchronizer').default;
|
||||
const { FileApiDriverAmazonS3 } = require('./file-api-driver-amazon-s3.js');
|
||||
const { S3Client, HeadBucketCommand } = require('@aws-sdk/client-s3');
|
||||
const S3 = require('aws-sdk/clients/s3');
|
||||
|
||||
class SyncTargetAmazonS3 extends BaseSyncTarget {
|
||||
static id() {
|
||||
@ -38,14 +38,10 @@ class SyncTargetAmazonS3 extends BaseSyncTarget {
|
||||
|
||||
s3AuthParameters() {
|
||||
return {
|
||||
// We need to set a region. See https://github.com/aws/aws-sdk-js-v3/issues/1845#issuecomment-754832210
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: Setting.value('sync.8.username'),
|
||||
secretAccessKey: Setting.value('sync.8.password'),
|
||||
},
|
||||
UseArnRegion: true, // override the request region with the region inferred from requested resource's ARN
|
||||
forcePathStyle: true,
|
||||
accessKeyId: Setting.value('sync.8.username'),
|
||||
secretAccessKey: Setting.value('sync.8.password'),
|
||||
s3UseArnRegion: true, // override the request region with the region inferred from requested resource's ARN
|
||||
s3ForcePathStyle: true,
|
||||
endpoint: Setting.value('sync.8.url'),
|
||||
};
|
||||
}
|
||||
@ -53,31 +49,20 @@ class SyncTargetAmazonS3 extends BaseSyncTarget {
|
||||
api() {
|
||||
if (this.api_) return this.api_;
|
||||
|
||||
this.api_ = new S3Client(this.s3AuthParameters());
|
||||
|
||||
// There is a bug with auto skew correction in aws-sdk-js-v3
|
||||
// and this attempts to remove the skew correction for all calls.
|
||||
// There are some additional spots in the app where we reset this
|
||||
// to zero as well as it appears the skew logic gets triggered
|
||||
// which makes "RequestTimeTooSkewed" errors...
|
||||
// See https://github.com/aws/aws-sdk-js-v3/issues/2208
|
||||
this.api_.config.systemClockOffset = 0;
|
||||
this.api_ = new S3(this.s3AuthParameters());
|
||||
return this.api_;
|
||||
}
|
||||
|
||||
static async newFileApi_(syncTargetId, options) {
|
||||
const apiOptions = {
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: options.username(),
|
||||
secretAccessKey: options.password(),
|
||||
},
|
||||
UseArnRegion: true, // override the request region with the region inferred from requested resource's ARN
|
||||
forcePathStyle: true,
|
||||
accessKeyId: options.username(),
|
||||
secretAccessKey: options.password(),
|
||||
s3UseArnRegion: true,
|
||||
s3ForcePathStyle: true,
|
||||
endpoint: options.url(),
|
||||
};
|
||||
|
||||
const api = new S3Client(apiOptions);
|
||||
const api = new S3(apiOptions);
|
||||
const driver = new FileApiDriverAmazonS3(api, SyncTargetAmazonS3.s3BucketName());
|
||||
const fileApi = new FileApi('', driver);
|
||||
fileApi.setSyncTargetId(syncTargetId);
|
||||
@ -95,14 +80,12 @@ class SyncTargetAmazonS3 extends BaseSyncTarget {
|
||||
|
||||
try {
|
||||
const headBucketReq = new Promise((resolve, reject) => {
|
||||
fileApi.driver().api().send(
|
||||
|
||||
new HeadBucketCommand({
|
||||
Bucket: options.path(),
|
||||
}),(err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
fileApi.driver().api().headBucket({
|
||||
Bucket: options.path(),
|
||||
},(err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
});
|
||||
const result = await headBucketReq;
|
||||
if (!result) throw new Error(`AWS S3 bucket not found: ${SyncTargetAmazonS3.s3BucketName()}`);
|
||||
|
||||
@ -85,6 +85,7 @@ shared.saveSettings = function(comp) {
|
||||
for (const key in comp.state.settings) {
|
||||
if (!comp.state.settings.hasOwnProperty(key)) continue;
|
||||
if (comp.state.changedSettingKeys.indexOf(key) < 0) continue;
|
||||
console.info('Saving', key, comp.state.settings[key]);
|
||||
Setting.setValue(key, comp.state.settings[key]);
|
||||
}
|
||||
|
||||
|
||||
@ -3,8 +3,6 @@ const { basename } = require('./path-utils');
|
||||
const shim = require('./shim').default;
|
||||
const JoplinError = require('./JoplinError').default;
|
||||
const { Buffer } = require('buffer');
|
||||
const { GetObjectCommand, ListObjectsV2Command, HeadObjectCommand, PutObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, CopyObjectCommand } = require('@aws-sdk/client-s3');
|
||||
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
|
||||
const S3_MAX_DELETES = 1000;
|
||||
|
||||
@ -28,33 +26,31 @@ class FileApiDriverAmazonS3 {
|
||||
}
|
||||
|
||||
hasErrorCode_(error, errorCode) {
|
||||
if (!error || typeof error.name !== 'string') return false;
|
||||
return error.name.indexOf(errorCode) >= 0;
|
||||
if (!error || typeof error.code !== 'string') return false;
|
||||
return error.code.indexOf(errorCode) >= 0;
|
||||
}
|
||||
|
||||
// Because of the way AWS-SDK-v3 works for getting data from a bucket we will
|
||||
// use a pre-signed URL to avoid https://github.com/aws/aws-sdk-js-v3/issues/1877
|
||||
async s3GenerateGetURL(key) {
|
||||
const signedUrl = await getSignedUrl(this.api(), new GetObjectCommand({
|
||||
Bucket: this.s3_bucket_,
|
||||
Key: key,
|
||||
}), {
|
||||
expiresIn: 3600,
|
||||
});
|
||||
return signedUrl;
|
||||
}
|
||||
|
||||
// We've now moved to aws-sdk-v3 and this note is outdated, but explains the promise structure.
|
||||
// Need to make a custom promise, built-in promise is broken: https://github.com/aws/aws-sdk-js/issues/1436
|
||||
// TODO: Re-factor to https://github.com/aws/aws-sdk-js-v3/tree/main/clients/client-s3#asyncawait
|
||||
async s3GetObject(key) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.api().getObject({
|
||||
Bucket: this.s3_bucket_,
|
||||
Key: key,
|
||||
}, (err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async s3ListObjects(key, cursor) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.api().send(new ListObjectsV2Command({
|
||||
this.api().listObjectsV2({
|
||||
Bucket: this.s3_bucket_,
|
||||
Prefix: key,
|
||||
Delimiter: '/',
|
||||
ContinuationToken: cursor,
|
||||
}), (err, response) => {
|
||||
}, (err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
@ -63,10 +59,10 @@ class FileApiDriverAmazonS3 {
|
||||
|
||||
async s3HeadObject(key) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.api().send(new HeadObjectCommand({
|
||||
this.api().headObject({
|
||||
Bucket: this.s3_bucket_,
|
||||
Key: key,
|
||||
}), (err, response) => {
|
||||
}, (err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
@ -75,11 +71,11 @@ class FileApiDriverAmazonS3 {
|
||||
|
||||
async s3PutObject(key, body) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.api().send(new PutObjectCommand({
|
||||
this.api().putObject({
|
||||
Bucket: this.s3_bucket_,
|
||||
Key: key,
|
||||
Body: body,
|
||||
}), (err, response) => {
|
||||
}, (err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
@ -91,12 +87,12 @@ class FileApiDriverAmazonS3 {
|
||||
const body = await shim.fsDriver().readFile(path, 'base64');
|
||||
const fileStat = await shim.fsDriver().stat(path);
|
||||
return new Promise((resolve, reject) => {
|
||||
this.api().send(new PutObjectCommand({
|
||||
this.api().putObject({
|
||||
Bucket: this.s3_bucket_,
|
||||
Key: key,
|
||||
Body: Buffer.from(body, 'base64'),
|
||||
ContentLength: `${fileStat.size}`,
|
||||
}), (err, response) => {
|
||||
}, (err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
@ -105,10 +101,10 @@ class FileApiDriverAmazonS3 {
|
||||
|
||||
async s3DeleteObject(key) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.api().send(new DeleteObjectCommand({
|
||||
this.api().deleteObject({
|
||||
Bucket: this.s3_bucket_,
|
||||
Key: key,
|
||||
}),
|
||||
},
|
||||
(err, response) => {
|
||||
if (err) {
|
||||
console.log(err.code);
|
||||
@ -122,10 +118,10 @@ class FileApiDriverAmazonS3 {
|
||||
// Assumes key is formatted, like `{Key: 's3 path'}`
|
||||
async s3DeleteObjects(keys) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.api().send(new DeleteObjectsCommand({
|
||||
this.api().deleteObjects({
|
||||
Bucket: this.s3_bucket_,
|
||||
Delete: { Objects: keys },
|
||||
}),
|
||||
},
|
||||
(err, response) => {
|
||||
if (err) {
|
||||
console.log(err.code);
|
||||
@ -192,20 +188,8 @@ class FileApiDriverAmazonS3 {
|
||||
prefixPath = `${prefixPath}/`;
|
||||
}
|
||||
|
||||
// There is a bug/quirk of aws-sdk-js-v3 which causes the
|
||||
// S3Client systemClockOffset to be wildly inaccurate. This
|
||||
// effectively removes the offset and sets it to system time.
|
||||
// See https://github.com/aws/aws-sdk-js-v3/issues/2208 for more.
|
||||
// If the user's time actaully off, then this should correctly
|
||||
// result in a RequestTimeTooSkewed error from s3ListObjects.
|
||||
this.api().config.systemClockOffset = 0;
|
||||
|
||||
let response = await this.s3ListObjects(prefixPath);
|
||||
|
||||
// In aws-sdk-js-v3 if there are no contents it no longer returns
|
||||
// an empty array. This creates an Empty array to pass onward.
|
||||
if (response.Contents === undefined) response.Contents = [];
|
||||
|
||||
let output = this.metadataToStats_(response.Contents, prefixPath);
|
||||
|
||||
while (response.IsTruncated) {
|
||||
@ -228,17 +212,31 @@ class FileApiDriverAmazonS3 {
|
||||
|
||||
try {
|
||||
let output = null;
|
||||
let response = null;
|
||||
|
||||
const s3Url = await this.s3GenerateGetURL(remotePath);
|
||||
const response = await this.s3GetObject(remotePath);
|
||||
output = response.Body;
|
||||
|
||||
if (options.target === 'file') {
|
||||
output = await shim.fetchBlob(s3Url, options);
|
||||
const filePath = options.path;
|
||||
if (!filePath) throw new Error('get: target options.path is missing');
|
||||
|
||||
// TODO: check if this ever hits on RN
|
||||
await shim.fsDriver().writeBinaryFile(filePath, output);
|
||||
return {
|
||||
ok: true,
|
||||
path: filePath,
|
||||
text: () => {
|
||||
return response.statusMessage;
|
||||
},
|
||||
json: () => {
|
||||
return { message: `${response.statusCode}: ${response.statusMessage}` };
|
||||
},
|
||||
status: response.statusCode,
|
||||
headers: response.headers,
|
||||
};
|
||||
}
|
||||
|
||||
if (responseFormat === 'text') {
|
||||
response = await shim.fetch(s3Url, options);
|
||||
output = await response.text();
|
||||
output = output.toString();
|
||||
}
|
||||
|
||||
return output;
|
||||
@ -312,11 +310,11 @@ class FileApiDriverAmazonS3 {
|
||||
|
||||
async move(oldPath, newPath) {
|
||||
const req = new Promise((resolve, reject) => {
|
||||
this.api().send(new CopyObjectCommand({
|
||||
this.api().copyObject({
|
||||
Bucket: this.s3_bucket_,
|
||||
CopySource: this.makePath_(oldPath),
|
||||
Key: newPath,
|
||||
}),(err, response) => {
|
||||
},(err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
@ -342,10 +340,10 @@ class FileApiDriverAmazonS3 {
|
||||
async clearRoot() {
|
||||
const listRecursive = async (cursor) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
return this.api().send(new ListObjectsV2Command({
|
||||
return this.api().listObjectsV2({
|
||||
Bucket: this.s3_bucket_,
|
||||
ContinuationToken: cursor,
|
||||
}), (err, response) => {
|
||||
}, (err, response) => {
|
||||
if (err) reject(err);
|
||||
else resolve(response);
|
||||
});
|
||||
@ -353,9 +351,6 @@ class FileApiDriverAmazonS3 {
|
||||
};
|
||||
|
||||
let response = await listRecursive();
|
||||
// In aws-sdk-js-v3 if there are no contents it no longer returns
|
||||
// an empty array. This creates an Empty array to pass onward.
|
||||
if (response.Contents === undefined) response.Contents = [];
|
||||
let keys = response.Contents.map((content) => content.Key);
|
||||
|
||||
while (response.IsTruncated) {
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
class FsDriverDummy {
|
||||
constructor() {}
|
||||
appendFileSync() {}
|
||||
writeBinaryFile() {}
|
||||
readFile() {}
|
||||
}
|
||||
|
||||
|
||||
@ -26,6 +26,16 @@ export default class FsDriverNode extends FsDriverBase {
|
||||
}
|
||||
}
|
||||
|
||||
public async writeBinaryFile(path: string, content: any) {
|
||||
try {
|
||||
// let buffer = new Buffer(content);
|
||||
const buffer = Buffer.from(content);
|
||||
return await fs.writeFile(path, buffer);
|
||||
} catch (error) {
|
||||
throw this.fsErrorToJsError_(error, path);
|
||||
}
|
||||
}
|
||||
|
||||
public async writeFile(path: string, string: string, encoding: string = 'base64') {
|
||||
try {
|
||||
if (encoding === 'buffer') {
|
||||
|
||||
@ -245,6 +245,10 @@ export default class Resource extends BaseItem {
|
||||
return this.fsDriver().readFile(this.fullPath(resource), 'Buffer');
|
||||
}
|
||||
|
||||
static setContent(resource: ResourceEntity, content: any) {
|
||||
return this.fsDriver().writeBinaryFile(this.fullPath(resource), content);
|
||||
}
|
||||
|
||||
static isResourceUrl(url: string) {
|
||||
return url && url.length === 34 && url[0] === ':' && url[1] === '/';
|
||||
}
|
||||
|
||||
1148
packages/lib/package-lock.json
generated
1148
packages/lib/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -25,14 +25,13 @@
|
||||
"typescript": "^4.0.5"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.22.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.23.0",
|
||||
"@joplin/fork-htmlparser2": "^4.1.31",
|
||||
"@joplin/fork-sax": "^1.2.35",
|
||||
"@joplin/renderer": "^2.2.3",
|
||||
"@joplin/turndown": "^4.0.53",
|
||||
"@joplin/turndown-plugin-gfm": "^1.0.35",
|
||||
"async-mutex": "^0.1.3",
|
||||
"aws-sdk": "^2.588.0",
|
||||
"base-64": "^0.1.0",
|
||||
"base64-stream": "^1.0.0",
|
||||
"builtin-modules": "^3.1.0",
|
||||
|
||||
@ -55,7 +55,7 @@ import SyncTargetJoplinCloud from '../SyncTargetJoplinCloud';
|
||||
import KeychainService from '../services/keychain/KeychainService';
|
||||
import { loadKeychainServiceAndSettings } from '../services/SettingUtils';
|
||||
const md5 = require('md5');
|
||||
const { S3Client } = require('@aws-sdk/client-s3');
|
||||
const S3 = require('aws-sdk/clients/s3');
|
||||
const { Dirnames } = require('../services/synchronizer/utils/types');
|
||||
|
||||
// Each suite has its own separate data and temp directory so that multiple
|
||||
@ -569,16 +569,10 @@ async function initFileApi() {
|
||||
const appDir = await api.appDirectory();
|
||||
fileApi = new FileApi(appDir, new FileApiDriverOneDrive(api));
|
||||
} else if (syncTargetId_ == SyncTargetRegistry.nameToId('amazon_s3')) {
|
||||
|
||||
// We make sure for S3 tests run in band because tests
|
||||
// share the same directory which will cause locking errors.
|
||||
|
||||
mustRunInBand();
|
||||
|
||||
const amazonS3CredsPath = `${oldTestDir}/support/amazon-s3-auth.json`;
|
||||
const amazonS3Creds = require(amazonS3CredsPath);
|
||||
if (!amazonS3Creds || !amazonS3Creds.accessKeyId) throw new Error(`AWS auth JSON missing in ${amazonS3CredsPath} format should be: { "accessKeyId": "", "secretAccessKey": "", "bucket": "mybucket"}`);
|
||||
const api = new S3Client({ region: 'us-east-1', accessKeyId: amazonS3Creds.accessKeyId, secretAccessKey: amazonS3Creds.secretAccessKey, s3UseArnRegion: true });
|
||||
const api = new S3({ accessKeyId: amazonS3Creds.accessKeyId, secretAccessKey: amazonS3Creds.secretAccessKey, s3UseArnRegion: true });
|
||||
fileApi = new FileApi('', new FileApiDriverAmazonS3(api, amazonS3Creds.bucket));
|
||||
} else if (syncTargetId_ == SyncTargetRegistry.nameToId('joplinServer')) {
|
||||
mustRunInBand();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user