self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

All: Add support for AWS S3 synchronisation (Beta) (#2815)

Browse Source
This commit is contained in:
alexchee 2020-07-15 05:22:55 -04:00 committed by GitHub
parent c8c4bb3245
commit 9a55afec01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 881 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -37,6 +37,7 @@ _mydocs
Assets/DownloadBadges*.psd
node_modules
Tools/github_oauth_token.txt
CliClient/tests/support/amazon-s3-auth.json
_releases
ReactNativeClient/lib/csstojs/
ReactNativeClient/lib/rnInjectedJs/

79
CliClient/package-lock.json generated
View File

@ -368,6 +368,34 @@
"resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
"integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
},
"aws-sdk": {
"version": "2.641.0",
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.641.0.tgz",
"integrity": "sha512-9GYrBWR7ygIwwFBr0L+P+6tecNGsDuSe1mB18rv7CXSDLDdg6VPYwma1PSw5bUBs4wix9ganK6QLfW8D8ztBEQ==",
"requires": {
"buffer": "4.9.1",
"events": "1.1.1",
"ieee754": "1.1.13",
"jmespath": "0.15.0",
"querystring": "0.2.0",
"sax": "1.2.1",
"url": "0.10.3",
"uuid": "3.3.2",
"xml2js": "0.4.19"
},
"dependencies": {
"sax": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
"integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
}
}
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
@ -455,6 +483,11 @@
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
"integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
},
"base64-js": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
"integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
},
"base64-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/base64-stream/-/base64-stream-1.0.0.tgz",
@ -573,6 +606,16 @@
"resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",
"integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow=="
},
"buffer": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
"integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4",
"isarray": "^1.0.0"
}
},
"buffer-equal": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz",
@ -1628,6 +1671,11 @@
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
"integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="
},
"events": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ="
},
"execa": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
@ -3137,6 +3185,11 @@
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
"integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
},
"ieee754": {
"version": "1.1.13",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
},
"ignore-walk": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
@ -3748,6 +3801,11 @@
"integrity": "sha512-nCeAiw37MIMA9w9IXso7bRaLl+c/ef3wnxsoSAlYrzS+Ot0zTG6nU8G/cIfGkqpkjX2wNaIW9RFG0TwIFnG6bA==",
"dev": true
},
"jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
},
"joplin-turndown": {
"version": "4.0.28",
"resolved": "https://registry.npmjs.org/joplin-turndown/-/joplin-turndown-4.0.28.tgz",
@ -5355,6 +5413,11 @@
"strict-uri-encode": "^1.0.0"
}
},
"querystring": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
"integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
},
"querystringify": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz",
@ -7033,6 +7096,22 @@
"resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz",
"integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI="
},
"url": {
"version": "0.10.3",
"resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
"integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
"requires": {
"punycode": "1.3.2",
"querystring": "0.2.0"
},
"dependencies": {
"punycode": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
}
}
},
"url-parse": {
"version": "1.4.7",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz",

1
CliClient/package.json
View File

@ -37,6 +37,7 @@
"dependencies": {
"app-module-path": "^2.2.0",
"async-mutex": "^0.1.3",
"aws-sdk": "^2.588.0",
"base-64": "^0.1.0",
"base64-stream": "^1.0.0",
"clean-html": "^1.5.0",

126
CliClient/tests/file_api_driver.js Normal file
View File

@ -0,0 +1,126 @@
/* eslint-disable no-unused-vars */
require('app-module-path').addPath(__dirname);
const { uuid } = require('lib/uuid.js');
const { time } = require('lib/time-utils.js');
const { asyncTest, sleep, fileApi, fileContentEqual, checkThrowAsync } = require('test-utils.js');
const { shim } = require('lib/shim.js');
const fs = require('fs-extra');
const Setting = require('lib/models/Setting.js');
jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000;
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
let api = null;
// To test out an FileApi implementation:
// * add a SyncTarget for your driver in `test-utils.js`
// * set `syncTargetId_` to your New SyncTarget:
// `const syncTargetId_ = SyncTargetRegistry.nameToId('memory');`
describe('fileApi', function() {
beforeEach(async (done) => {
api = new fileApi();
api.clearRoot();
done();
});
describe('list', function() {
it('should return items with relative path', asyncTest(async () => {
await api.mkdir('.subfolder');
await api.put('1', 'something on root 1');
await api.put('.subfolder/1', 'something subfolder 1');
await api.put('.subfolder/2', 'something subfolder 2');
await api.put('.subfolder/3', 'something subfolder 3');
sleep(0.8);
const response = await api.list('.subfolder');
const items = response.items;
expect(items.length).toBe(3);
expect(items[0].path).toBe('1');
}));
it('should default to only files on root directory', asyncTest(async () => {
await api.mkdir('.subfolder');
await api.put('.subfolder/1', 'something subfolder 1');
await api.put('file1', 'something 1');
await api.put('file2', 'something 2');
sleep(0.6);
const response = await api.list();
expect(response.items.length).toBe(2);
}));
}); // list
describe('delete', function() {
it('should not error if file does not exist', asyncTest(async () => {
const hasThrown = await checkThrowAsync(async () => await api.delete('nonexistant_file'));
expect(hasThrown).toBe(false);
}));
it('should delete specific file given full path', asyncTest(async () => {
await api.mkdir('deleteDir');
await api.put('deleteDir/1', 'something 1');
await api.put('deleteDir/2', 'something 2');
sleep(0.4);
await api.delete('deleteDir/1');
let response = await api.list('deleteDir');
expect(response.items.length).toBe(1);
response = await api.list('deleteDir/1');
expect(response.items.length).toBe(0);
}));
}); // delete
describe('get', function() {
it('should return null if object does not exist', asyncTest(async () => {
const response = await api.get('nonexistant_file');
expect(response).toBe(null);
}));
it('should return UTF-8 encoded string by default', asyncTest(async () => {
await api.put('testnote.md', 'something 2');
const response = await api.get('testnote.md');
expect(response).toBe('something 2');
}));
it('should return a Response object and writes file to options.path, if options.target is "file"', asyncTest(async () => {
const localFilePath = `${Setting.value('tempDir')}/${uuid.create()}.md`;
await api.put('testnote.md', 'something 2');
sleep(0.2);
const response = await api.get('testnote.md', { target: 'file', path: localFilePath });
expect(typeof response).toBe('object');
// expect(response.path).toBe(localFilePath);
expect(fs.existsSync(localFilePath)).toBe(true);
expect(fs.readFileSync(localFilePath, 'utf8')).toBe('something 2');
}));
}); // get
describe('put', function() {
it('should create file to remote path and content', asyncTest(async () => {
await api.put('putTest.md', 'I am your content');
sleep(0.2);
const response = await api.get('putTest.md');
expect(response).toBe('I am your content');
}));
it('should upload file in options.path to remote path, if options.source is "file"', asyncTest(async () => {
const localFilePath = `${Setting.value('tempDir')}/${uuid.create()}.md`;
fs.writeFileSync(localFilePath, 'I am the local file.');
await api.put('testfile', 'ignore me', { source: 'file', path: localFilePath });
sleep(0.2);
const response = await api.get('testfile');
expect(response).toBe('I am the local file.');
}));
}); // put
});

12
CliClient/tests/test-utils.js
View File

@ -21,6 +21,7 @@ const { FileApiDriverMemory } = require('lib/file-api-driver-memory.js');
const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
const { FileApiDriverWebDav } = require('lib/file-api-driver-webdav.js');
const { FileApiDriverDropbox } = require('lib/file-api-driver-dropbox.js');
const { FileApiDriverAmazonS3 } = require('lib/file-api-driver-amazon-s3.js');
const BaseService = require('lib/services/BaseService.js');
const { FsDriverNode } = require('lib/fs-driver-node.js');
const { time } = require('lib/time-utils.js');
@ -33,6 +34,7 @@ const SyncTargetFilesystem = require('lib/SyncTargetFilesystem.js');
const SyncTargetOneDrive = require('lib/SyncTargetOneDrive.js');
const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js');
const SyncTargetDropbox = require('lib/SyncTargetDropbox.js');
const SyncTargetAmazonS3 = require('lib/SyncTargetAmazonS3.js');
const EncryptionService = require('lib/services/EncryptionService.js');
const DecryptionWorker = require('lib/services/DecryptionWorker.js');
const ResourceService = require('lib/services/ResourceService.js');
@ -45,6 +47,7 @@ const { loadKeychainServiceAndSettings } = require('lib/services/SettingUtils');
const KeychainServiceDriver = require('lib/services/keychain/KeychainServiceDriver.node').default;
const KeychainServiceDriverDummy = require('lib/services/keychain/KeychainServiceDriver.dummy').default;
const md5 = require('md5');
const S3 = require('aws-sdk/clients/s3');
const databases_ = [];
const synchronizers_ = [];
@ -83,11 +86,13 @@ SyncTargetRegistry.addClass(SyncTargetFilesystem);
SyncTargetRegistry.addClass(SyncTargetOneDrive);
SyncTargetRegistry.addClass(SyncTargetNextcloud);
SyncTargetRegistry.addClass(SyncTargetDropbox);
SyncTargetRegistry.addClass(SyncTargetAmazonS3);
// const syncTargetId_ = SyncTargetRegistry.nameToId("nextcloud");
const syncTargetId_ = SyncTargetRegistry.nameToId('memory');
// const syncTargetId_ = SyncTargetRegistry.nameToId('filesystem');
// const syncTargetId_ = SyncTargetRegistry.nameToId('dropbox');
// const syncTargetId_ = SyncTargetRegistry.nameToId('amazon_s3');
const syncDir = `${__dirname}/../tests/sync`;
const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 100;// 400;
@ -351,8 +356,15 @@ function fileApi() {
if (!authToken) throw new Error(`Dropbox auth token missing in ${authTokenPath}`);
api.setAuthToken(authToken);
fileApi_ = new FileApi('', new FileApiDriverDropbox(api));
} else if (syncTargetId_ == SyncTargetRegistry.nameToId('amazon_s3')) {
const amazonS3CredsPath = `${__dirname}/support/amazon-s3-auth.json`;
const amazonS3Creds = require(amazonS3CredsPath);
if (!amazonS3Creds || !amazonS3Creds.accessKeyId) throw new Error(`AWS auth JSON missing in ${amazonS3CredsPath} format should be: { "accessKeyId": "", "secretAccessKey": "", "bucket": "mybucket"}`);
const api = new S3({ accessKeyId: amazonS3Creds.accessKeyId, secretAccessKey: amazonS3Creds.secretAccessKey, s3UseArnRegion: true });
fileApi_ = new FileApi('', new FileApiDriverAmazonS3(api, amazonS3Creds.bucket));
}
fileApi_.setLogger(logger);
fileApi_.setSyncTargetId(syncTargetId_);
fileApi_.requestRepeatCount_ = 0;

77
ElectronClient/package-lock.json generated
View File

@ -715,6 +715,34 @@
"resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
"integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
},
"aws-sdk": {
"version": "2.680.0",
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.680.0.tgz",
"integrity": "sha512-sq19d5cNrgtcoMQc8GlwRrN11zT5FVxc+ZHL9P6lNAlGA3av3dwpt6+4smvhHpPzpzT0fG5A7HMczgjbLaLUDA==",
"requires": {
"buffer": "4.9.1",
"events": "1.1.1",
"ieee754": "1.1.13",
"jmespath": "0.15.0",
"querystring": "0.2.0",
"sax": "1.2.1",
"url": "0.10.3",
"uuid": "3.3.2",
"xml2js": "0.4.19"
},
"dependencies": {
"sax": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
"integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
}
}
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
@ -1946,6 +1974,15 @@
"readable-stream": "^3.4.0"
},
"dependencies": {
"buffer": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
"integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4"
}
},
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@ -2132,12 +2169,13 @@
"integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow=="
},
"buffer": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
"integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
"integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4"
"ieee754": "^1.1.4",
"isarray": "^1.0.0"
}
},
"buffer-crc32": {
@ -4334,6 +4372,11 @@
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz",
"integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs="
},
"events": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ="
},
"execa": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
@ -6852,6 +6895,11 @@
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
},
"joplin-turndown": {
"version": "4.0.28",
"resolved": "https://registry.npmjs.org/joplin-turndown/-/joplin-turndown-4.0.28.tgz",
@ -9506,6 +9554,11 @@
"strict-uri-encode": "^1.0.0"
}
},
"querystring": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
"integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
},
"querystringify": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.0.tgz",
@ -11660,6 +11713,22 @@
"resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz",
"integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI="
},
"url": {
"version": "0.10.3",
"resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
"integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
"requires": {
"punycode": "1.3.2",
"querystring": "0.2.0"
},
"dependencies": {
"punycode": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
}
}
},
"url-parse": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.4.tgz",

1
ElectronClient/package.json
View File

@ -99,6 +99,7 @@
"@fortawesome/fontawesome-free": "^5.13.0",
"app-module-path": "^2.2.0",
"async-mutex": "^0.1.3",
"aws-sdk": "^2.594.0",
"base-64": "^0.1.0",
"base64-stream": "^1.0.0",
"chokidar": "^3.0.0",

2
ReactNativeClient/lib/BaseApplication.js
View File

@ -29,6 +29,7 @@ const SyncTargetOneDriveDev = require('lib/SyncTargetOneDriveDev.js');
const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js');
const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js');
const SyncTargetDropbox = require('lib/SyncTargetDropbox.js');
const SyncTargetAmazonS3 = require('lib/SyncTargetAmazonS3.js');
const EncryptionService = require('lib/services/EncryptionService');
const ResourceFetcher = require('lib/services/ResourceFetcher');
const SearchEngineUtils = require('lib/services/SearchEngineUtils');
@ -629,6 +630,7 @@ class BaseApplication {
SyncTargetRegistry.addClass(SyncTargetNextcloud);
SyncTargetRegistry.addClass(SyncTargetWebDAV);
SyncTargetRegistry.addClass(SyncTargetDropbox);
SyncTargetRegistry.addClass(SyncTargetAmazonS3);
await shim.fsDriver().remove(tempDir);

110
ReactNativeClient/lib/SyncTargetAmazonS3.js Normal file
View File

@ -0,0 +1,110 @@
const BaseSyncTarget = require('lib/BaseSyncTarget.js');
const { _ } = require('lib/locale.js');
const Setting = require('lib/models/Setting.js');
const { FileApi } = require('lib/file-api.js');
const { Synchronizer } = require('lib/synchronizer.js');
const { FileApiDriverAmazonS3 } = require('lib/file-api-driver-amazon-s3.js');
const S3 = require('aws-sdk/clients/s3');
class SyncTargetAmazonS3 extends BaseSyncTarget {
static id() {
return 8;
}
static supportsConfigCheck() {
return true;
}
constructor(db, options = null) {
super(db, options);
this.api_ = null;
}
static targetName() {
return 'amazon_s3';
}
static label() {
return _('AWS S3') + ' (Beta)';
}
async isAuthenticated() {
return true;
}
static s3BucketName() {
return Setting.value('sync.8.path');
}
s3AuthParameters() {
return {
accessKeyId: Setting.value('sync.8.username'),
secretAccessKey: Setting.value('sync.8.password'),
s3UseArnRegion: true, // override the request region with the region inferred from requested resource's ARN
};
}
api() {
if (this.api_) return this.api_;
this.api_ = new S3(this.s3AuthParameters());
return this.api_;
}
static async newFileApi_(syncTargetId, options) {
const apiOptions = {
accessKeyId: options.username(),
secretAccessKey: options.password(),
s3UseArnRegion: true,
};
const api = new S3(apiOptions);
const driver = new FileApiDriverAmazonS3(api, SyncTargetAmazonS3.s3BucketName());
const fileApi = new FileApi('', driver);
fileApi.setSyncTargetId(syncTargetId);
return fileApi;
}
static async checkConfig(options) {
const fileApi = await SyncTargetAmazonS3.newFileApi_(SyncTargetAmazonS3.id(), options);
fileApi.requestRepeatCount_ = 0;
const output = {
ok: false,
errorMessage: '',
};
try {
const headBucketReq = new Promise((resolve, reject) => {
fileApi.driver().api().headBucket({
Bucket: options.path(),
},(err, response) => {
if (err) reject(err);
else resolve(response);
});
});
const result = await headBucketReq;
if (!result) throw new Error(`AWS S3 bucket not found: ${SyncTargetAmazonS3.s3BucketName()}`);
output.ok = true;
} catch (error) {
output.errorMessage = error.message;
if (error.code) output.errorMessage += ` (Code ${error.code})`;
}
return output;
}
async initFileApi() {
const appDir = '';
const fileApi = new FileApi(appDir, new FileApiDriverAmazonS3(this.api(), SyncTargetAmazonS3.s3BucketName()));
fileApi.setSyncTargetId(SyncTargetAmazonS3.id());
return fileApi;
}
async initSynchronizer() {
return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType'));
}
}
module.exports = SyncTargetAmazonS3;

361
ReactNativeClient/lib/file-api-driver-amazon-s3.js Normal file
View File

@ -0,0 +1,361 @@
const { basicDelta } = require('lib/file-api');
const { basename } = require('lib/path-utils');
const { shim } = require('lib/shim');
const JoplinError = require('lib/JoplinError');
const S3_MAX_DELETES = 1000;
class FileApiDriverAmazonS3 {
constructor(api, s3_bucket) {
this.s3_bucket_ = s3_bucket;
this.api_ = api;
}
api() {
return this.api_;
}
requestRepeatCount() {
return 3;
}
makePath_(path) {
if (!path) return '';
return path;
}
hasErrorCode_(error, errorCode) {
if (!error || typeof error.code !== 'string') return false;
return error.code.indexOf(errorCode) >= 0;
}
// Need to make a custom promise, built-in promise is broken: https://github.com/aws/aws-sdk-js/issues/1436
async s3GetObject(key) {
return new Promise((resolve, reject) => {
this.api().getObject({
Bucket: this.s3_bucket_,
Key: key,
}, (err, response) => {
if (err) reject(err);
else resolve(response);
});
});
}
async s3ListObjects(key, cursor) {
return new Promise((resolve, reject) => {
this.api().listObjectsV2({
Bucket: this.s3_bucket_,
Prefix: key,
Delimiter: '/',
ContinuationToken: cursor,
}, (err, response) => {
if (err) reject(err);
else resolve(response);
});
});
}
async s3HeadObject(key) {
return new Promise((resolve, reject) => {
this.api().headObject({
Bucket: this.s3_bucket_,
Key: key,
}, (err, response) => {
if (err) reject(err);
else resolve(response);
});
});
}
async s3PutObject(key, body) {
return new Promise((resolve, reject) => {
this.api().putObject({
Bucket: this.s3_bucket_,
Key: key,
Body: body,
}, (err, response) => {
if (err) reject(err);
else resolve(response);
});
});
}
async s3UploadFileFrom(path, key) {
if (!shim.fsDriver().exists(path)) throw new Error('s3UploadFileFrom: file does not exist');
const body = await shim.fsDriver().readFile(path, 'Buffer');
return new Promise((resolve, reject) => {
this.api().upload({
Bucket: this.s3_bucket_,
Key: key,
Body: body,
}, (err, response) => {
if (err) reject(err);
else resolve(response);
});
});
}
async s3DeleteObject(key) {
return new Promise((resolve, reject) => {
this.api().deleteObject({
Bucket: this.s3_bucket_,
Key: key,
},
(err, response) => {
if (err) {
console.log(err.code);
console.log(err.message);
reject(err);
} else { resolve(response); }
});
});
}
// Assumes key is formatted, like `{Key: 's3 path'}`
async s3DeleteObjects(keys) {
return new Promise((resolve, reject) => {
this.api().deleteObjects({
Bucket: this.s3_bucket_,
Delete: { Objects: keys },
},
(err, response) => {
if (err) {
console.log(err.code);
console.log(err.message);
reject(err);
} else { resolve(response); }
});
});
}
async stat(path) {
try {
const metadata = await this.s3HeadObject(this.makePath_(path));
return this.metadataToStat_(metadata, path);
} catch (error) {
if (this.hasErrorCode_(error, 'NotFound')) {
// ignore
} else {
throw error;
}
}
}
metadataToStat_(md, path) {
const relativePath = basename(path);
const output = {
path: relativePath,
updated_time: md['LastModified'] ? new Date(md['LastModified']) : new Date(),
isDeleted: !!md['DeleteMarker'],
isDir: false,
};
return output;
}
metadataToStats_(mds) {
const output = [];
for (let i = 0; i < mds.length; i++) {
output.push(this.metadataToStat_(mds[i], mds[i].Key));
}
return output;
}
async setTimestamp() {
throw new Error('Not implemented'); // Not needed anymore
}
async delta(path, options) {
const getDirStats = async path => {
const result = await this.list(path);
return result.items;
};
return await basicDelta(path, getDirStats, options);
}
async list(path) {
let prefixPath = this.makePath_(path);
const pathLen = prefixPath.length;
if (pathLen > 0 && prefixPath[pathLen - 1] !== '/') {
prefixPath = `${prefixPath}/`;
}
let response = await this.s3ListObjects(prefixPath);
let output = this.metadataToStats_(response.Contents, prefixPath);
while (response.IsTruncated) {
response = await this.s3ListObjects(prefixPath, response.NextContinuationToken);
output = output.concat(this.metadataToStats_(response.Contents, prefixPath));
}
return {
items: output,
hasMore: false,
context: { cursor: response.NextContinuationToken },
};
}
async get(path, options) {
const remotePath = this.makePath_(path);
if (!options) options = {};
const responseFormat = options.responseFormat || 'text';
try {
let output = null;
const response = await this.s3GetObject(remotePath);
output = response.Body;
if (options.target === 'file') {
const filePath = options.path;
if (!filePath) throw new Error('get: target options.path is missing');
// TODO: check if this ever hits on RN
await shim.fsDriver().writeBinaryFile(filePath, output);
return {
ok: true,
path: filePath,
text: () => {
return response.statusMessage;
},
json: () => {
return { message: `${response.statusCode}: ${response.statusMessage}` };
},
status: response.statusCode,
headers: response.headers,
};
}
if (responseFormat === 'text') {
output = output.toString();
}
return output;
} catch (error) {
if (this.hasErrorCode_(error, 'NoSuchKey')) {
return null;
} else if (this.hasErrorCode_(error, 'AccessDenied')) {
throw new JoplinError('Do not have proper permissions to Bucket', 'rejectedByTarget');
} else {
throw error;
}
}
}
// Don't need to make directories, S3 is key based storage.
async mkdir() {
return true;
}
async put(path, content, options = null) {
const remotePath = this.makePath_(path);
if (!options) options = {};
// See https://github.com/facebook/react-native/issues/14445#issuecomment-352965210
if (typeof content === 'string') content = shim.Buffer.from(content, 'utf8');
try {
if (options.source === 'file') {
await this.s3UploadFileFrom(options.path, remotePath);
return;
}
await this.s3PutObject(remotePath, content);
} catch (error) {
if (this.hasErrorCode_(error, 'AccessDenied')) {
throw new JoplinError('Do not have proper permissions to Bucket', 'rejectedByTarget');
} else {
throw error;
}
}
}
async delete(path) {
try {
await this.s3DeleteObject(this.makePath_(path));
} catch (error) {
if (this.hasErrorCode_(error, 'NoSuchKey')) {
// ignore
} else {
throw error;
}
}
}
async batchDeletes(paths) {
const keys = paths.map(path => { return { Key: path }; });
while (keys.length > 0) {
const toDelete = keys.splice(0, S3_MAX_DELETES);
try {
await this.s3DeleteObjects(toDelete);
} catch (error) {
if (this.hasErrorCode_(error, 'NoSuchKey')) {
// ignore
} else {
throw error;
}
}
}
}
async move(oldPath, newPath) {
const req = new Promise((resolve, reject) => {
this.api().copyObject({
Bucket: this.s3_bucket_,
CopySource: this.makePath_(oldPath),
Key: newPath,
},(err, response) => {
if (err) reject(err);
else resolve(response);
});
});
try {
await req;
this.delete(oldPath);
} catch (error) {
if (this.hasErrorCode_(error, 'NoSuchKey')) {
// ignore
} else {
throw error;
}
}
}
format() {
throw new Error('Not supported');
}
async clearRoot() {
const listRecursive = async (cursor) => {
return new Promise((resolve, reject) => {
return this.api().listObjectsV2({
Bucket: this.s3_bucket_,
ContinuationToken: cursor,
}, (err, response) => {
if (err) reject(err);
else resolve(response);
});
});
};
let response = await listRecursive();
let keys = response.Contents.map((content) => content.Key);
while (response.IsTruncated) {
response = await listRecursive(response.NextContinuationToken);
keys = keys.concat(response.Contents.map((content) => content.Key));
}
this.batchDeletes(keys);
}
}
module.exports = { FileApiDriverAmazonS3 };

1
ReactNativeClient/lib/file-api.js
View File

@ -172,6 +172,7 @@ class FileApi {
// });
}
// Returns UTF-8 encoded string by default, or a Response if `options.target = 'file'`
get(path, options = null) {
if (!options) options = {};
if (!options.encoding) options.encoding = 'utf8';

61
ReactNativeClient/lib/models/Setting.js
View File

@ -171,16 +171,45 @@ class Setting extends BaseModel {
secure: true,
},
'sync.3.auth': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.4.auth': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.7.auth': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.1.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.2.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.3.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.4.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.5.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.6.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.7.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.8.path': {
value: '',
type: Setting.TYPE_STRING,
section: 'sync',
show: settings => {
try {
return settings['sync.target'] == SyncTargetRegistry.nameToId('amazon_s3');
} catch (error) {
return false;
}
},
filter: value => {
return value ? rtrimSlashes(value) : '';
},
public: true,
label: () => _('AWS S3 bucket'),
description: () => emptyDirWarning,
},
'sync.8.username': {
value: '',
type: Setting.TYPE_STRING,
section: 'sync',
show: settings => {
return settings['sync.target'] == SyncTargetRegistry.nameToId('amazon_s3');
},
public: true,
label: () => _('AWS key'),
},
'sync.8.password': {
value: '',
type: Setting.TYPE_STRING,
section: 'sync',
show: settings => {
return settings['sync.target'] == SyncTargetRegistry.nameToId('amazon_s3');
},
public: true,
label: () => _('AWS secret'),
secure: true,
},
'sync.5.syncTargets': { value: {}, type: Setting.TYPE_OBJECT, public: false },
@ -203,6 +232,18 @@ class Setting extends BaseModel {
},
},
'sync.3.auth': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.4.auth': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.7.auth': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.1.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.2.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.3.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.4.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.5.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.6.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.7.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.8.context': { value: '', type: Setting.TYPE_STRING, public: false },
'sync.maxConcurrentConnections': { value: 5, type: Setting.TYPE_INT, public: true, advanced: true, section: 'sync', label: () => _('Max concurrent connections'), minimum: 1, maximum: 20, step: 1 },
activeFolderId: { value: '', type: Setting.TYPE_STRING, public: false },

62
ReactNativeClient/package-lock.json generated
View File

@ -2533,6 +2533,63 @@
"resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
"integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
},
"aws-sdk": {
"version": "2.642.0",
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.642.0.tgz",
"integrity": "sha512-0ZNgL1HBXRVobFD9Z64RyQk50cNABDMU1GV4lYIAvao4urYqYJi2MEVQmq+7WyXyzkBWu3lAPNDiJ8WW7emTzg==",
"requires": {
"buffer": "4.9.1",
"events": "1.1.1",
"ieee754": "1.1.13",
"jmespath": "0.15.0",
"querystring": "0.2.0",
"sax": "1.2.1",
"url": "0.10.3",
"uuid": "3.3.2",
"xml2js": "0.4.19"
},
"dependencies": {
"buffer": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
"integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4",
"isarray": "^1.0.0"
}
},
"ieee754": {
"version": "1.1.13",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
},
"punycode": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
},
"sax": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
"integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
},
"url": {
"version": "0.10.3",
"resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
"integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
"requires": {
"punycode": "1.3.2",
"querystring": "0.2.0"
}
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
}
}
},
"babel-code-frame": {
"version": "6.26.0",
"resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz",
@ -6246,6 +6303,11 @@
"resolved": "https://registry.npmjs.org/jetifier/-/jetifier-1.6.5.tgz",
"integrity": "sha512-T7yzBSu9PR+DqjYt+I0KVO1XTb1QhAfHnXV5Nd3xpbXM6Xg4e3vP60Q4qkNU8Fh6PHC2PivPUNN3rY7G2MxcDQ=="
},
"jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
},
"js-tokens": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz",

1
ReactNativeClient/package.json
View File

@ -18,6 +18,7 @@
"@react-native-community/push-notification-ios": "^1.0.5",
"@react-native-community/slider": "^2.0.8",
"async-mutex": "^0.1.3",
"aws-sdk": "^2.588.0",
"base-64": "^0.1.0",
"buffer": "^5.0.8",
"color": "^3.1.2",