From 9a55afec01fa9adcba4157ee09db20055dce3afa Mon Sep 17 00:00:00 2001 From: alexchee Date: Wed, 15 Jul 2020 05:22:55 -0400 Subject: [PATCH] All: Add support for AWS S3 synchronisation (Beta) (#2815) --- .gitignore | 1 + CliClient/package-lock.json | 79 ++++ CliClient/package.json | 1 + CliClient/tests/file_api_driver.js | 126 ++++++ CliClient/tests/test-utils.js | 12 + ElectronClient/package-lock.json | 77 +++- ElectronClient/package.json | 1 + ReactNativeClient/lib/BaseApplication.js | 2 + ReactNativeClient/lib/SyncTargetAmazonS3.js | 110 ++++++ .../lib/file-api-driver-amazon-s3.js | 361 ++++++++++++++++++ ReactNativeClient/lib/file-api.js | 1 + ReactNativeClient/lib/models/Setting.js | 61 ++- ReactNativeClient/package-lock.json | 62 +++ ReactNativeClient/package.json | 1 + 14 files changed, 881 insertions(+), 14 deletions(-) create mode 100644 CliClient/tests/file_api_driver.js create mode 100644 ReactNativeClient/lib/SyncTargetAmazonS3.js create mode 100644 ReactNativeClient/lib/file-api-driver-amazon-s3.js diff --git a/.gitignore b/.gitignore index a717657ad..48a7faad3 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ _mydocs Assets/DownloadBadges*.psd node_modules Tools/github_oauth_token.txt +CliClient/tests/support/amazon-s3-auth.json _releases ReactNativeClient/lib/csstojs/ ReactNativeClient/lib/rnInjectedJs/ diff --git a/CliClient/package-lock.json b/CliClient/package-lock.json index 433a66d55..0d47feefc 100644 --- a/CliClient/package-lock.json +++ b/CliClient/package-lock.json @@ -368,6 +368,34 @@ "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, + "aws-sdk": { + "version": "2.641.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.641.0.tgz", + "integrity": "sha512-9GYrBWR7ygIwwFBr0L+P+6tecNGsDuSe1mB18rv7CXSDLDdg6VPYwma1PSw5bUBs4wix9ganK6QLfW8D8ztBEQ==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -455,6 +483,11 @@ "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs=" }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, "base64-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/base64-stream/-/base64-stream-1.0.0.tgz", @@ -573,6 +606,16 @@ "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" }, + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "buffer-equal": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz", @@ -1628,6 +1671,11 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -3137,6 +3185,11 @@ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore-walk": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", @@ -3748,6 +3801,11 @@ "integrity": "sha512-nCeAiw37MIMA9w9IXso7bRaLl+c/ef3wnxsoSAlYrzS+Ot0zTG6nU8G/cIfGkqpkjX2wNaIW9RFG0TwIFnG6bA==", "dev": true }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "joplin-turndown": { "version": "4.0.28", "resolved": "https://registry.npmjs.org/joplin-turndown/-/joplin-turndown-4.0.28.tgz", @@ -5355,6 +5413,11 @@ "strict-uri-encode": "^1.0.0" } }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "querystringify": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz", @@ -7033,6 +7096,22 @@ "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=" }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, "url-parse": { "version": "1.4.7", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", diff --git a/CliClient/package.json b/CliClient/package.json index 80678b0e5..6a483f04b 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -37,6 +37,7 @@ "dependencies": { "app-module-path": "^2.2.0", "async-mutex": "^0.1.3", + "aws-sdk": "^2.588.0", "base-64": "^0.1.0", "base64-stream": "^1.0.0", "clean-html": "^1.5.0", diff --git a/CliClient/tests/file_api_driver.js b/CliClient/tests/file_api_driver.js new file mode 100644 index 000000000..2d714500a --- /dev/null +++ b/CliClient/tests/file_api_driver.js @@ -0,0 +1,126 @@ +/* eslint-disable no-unused-vars */ + +require('app-module-path').addPath(__dirname); + +const { uuid } = require('lib/uuid.js'); +const { time } = require('lib/time-utils.js'); +const { asyncTest, sleep, fileApi, fileContentEqual, checkThrowAsync } = require('test-utils.js'); +const { shim } = require('lib/shim.js'); +const fs = require('fs-extra'); +const Setting = require('lib/models/Setting.js'); + +jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000; + +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); +}); + +let api = null; + +// To test out an FileApi implementation: +// * add a SyncTarget for your driver in `test-utils.js` +// * set `syncTargetId_` to your New SyncTarget: +// `const syncTargetId_ = SyncTargetRegistry.nameToId('memory');` +describe('fileApi', function() { + + beforeEach(async (done) => { + api = new fileApi(); + api.clearRoot(); + done(); + }); + + describe('list', function() { + it('should return items with relative path', asyncTest(async () => { + await api.mkdir('.subfolder'); + await api.put('1', 'something on root 1'); + await api.put('.subfolder/1', 'something subfolder 1'); + await api.put('.subfolder/2', 'something subfolder 2'); + await api.put('.subfolder/3', 'something subfolder 3'); + sleep(0.8); + + const response = await api.list('.subfolder'); + const items = response.items; + expect(items.length).toBe(3); + expect(items[0].path).toBe('1'); + })); + + it('should default to only files on root directory', asyncTest(async () => { + await api.mkdir('.subfolder'); + await api.put('.subfolder/1', 'something subfolder 1'); + await api.put('file1', 'something 1'); + await api.put('file2', 'something 2'); + sleep(0.6); + + const response = await api.list(); + expect(response.items.length).toBe(2); + })); + }); // list + + describe('delete', function() { + it('should not error if file does not exist', asyncTest(async () => { + const hasThrown = await checkThrowAsync(async () => await api.delete('nonexistant_file')); + expect(hasThrown).toBe(false); + })); + + it('should delete specific file given full path', asyncTest(async () => { + await api.mkdir('deleteDir'); + await api.put('deleteDir/1', 'something 1'); + await api.put('deleteDir/2', 'something 2'); + sleep(0.4); + + await api.delete('deleteDir/1'); + let response = await api.list('deleteDir'); + expect(response.items.length).toBe(1); + response = await api.list('deleteDir/1'); + expect(response.items.length).toBe(0); + })); + }); // delete + + describe('get', function() { + it('should return null if object does not exist', asyncTest(async () => { + const response = await api.get('nonexistant_file'); + expect(response).toBe(null); + })); + + it('should return UTF-8 encoded string by default', asyncTest(async () => { + await api.put('testnote.md', 'something 2'); + + const response = await api.get('testnote.md'); + expect(response).toBe('something 2'); + })); + + it('should return a Response object and writes file to options.path, if options.target is "file"', asyncTest(async () => { + const localFilePath = `${Setting.value('tempDir')}/${uuid.create()}.md`; + await api.put('testnote.md', 'something 2'); + sleep(0.2); + + const response = await api.get('testnote.md', { target: 'file', path: localFilePath }); + expect(typeof response).toBe('object'); + // expect(response.path).toBe(localFilePath); + expect(fs.existsSync(localFilePath)).toBe(true); + expect(fs.readFileSync(localFilePath, 'utf8')).toBe('something 2'); + })); + }); // get + + describe('put', function() { + it('should create file to remote path and content', asyncTest(async () => { + await api.put('putTest.md', 'I am your content'); + sleep(0.2); + + const response = await api.get('putTest.md'); + expect(response).toBe('I am your content'); + })); + + it('should upload file in options.path to remote path, if options.source is "file"', asyncTest(async () => { + const localFilePath = `${Setting.value('tempDir')}/${uuid.create()}.md`; + fs.writeFileSync(localFilePath, 'I am the local file.'); + + await api.put('testfile', 'ignore me', { source: 'file', path: localFilePath }); + sleep(0.2); + + const response = await api.get('testfile'); + expect(response).toBe('I am the local file.'); + })); + }); // put + +}); diff --git a/CliClient/tests/test-utils.js b/CliClient/tests/test-utils.js index c1450e1eb..b08a07a58 100644 --- a/CliClient/tests/test-utils.js +++ b/CliClient/tests/test-utils.js @@ -21,6 +21,7 @@ const { FileApiDriverMemory } = require('lib/file-api-driver-memory.js'); const { FileApiDriverLocal } = require('lib/file-api-driver-local.js'); const { FileApiDriverWebDav } = require('lib/file-api-driver-webdav.js'); const { FileApiDriverDropbox } = require('lib/file-api-driver-dropbox.js'); +const { FileApiDriverAmazonS3 } = require('lib/file-api-driver-amazon-s3.js'); const BaseService = require('lib/services/BaseService.js'); const { FsDriverNode } = require('lib/fs-driver-node.js'); const { time } = require('lib/time-utils.js'); @@ -33,6 +34,7 @@ const SyncTargetFilesystem = require('lib/SyncTargetFilesystem.js'); const SyncTargetOneDrive = require('lib/SyncTargetOneDrive.js'); const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js'); const SyncTargetDropbox = require('lib/SyncTargetDropbox.js'); +const SyncTargetAmazonS3 = require('lib/SyncTargetAmazonS3.js'); const EncryptionService = require('lib/services/EncryptionService.js'); const DecryptionWorker = require('lib/services/DecryptionWorker.js'); const ResourceService = require('lib/services/ResourceService.js'); @@ -45,6 +47,7 @@ const { loadKeychainServiceAndSettings } = require('lib/services/SettingUtils'); const KeychainServiceDriver = require('lib/services/keychain/KeychainServiceDriver.node').default; const KeychainServiceDriverDummy = require('lib/services/keychain/KeychainServiceDriver.dummy').default; const md5 = require('md5'); +const S3 = require('aws-sdk/clients/s3'); const databases_ = []; const synchronizers_ = []; @@ -83,11 +86,13 @@ SyncTargetRegistry.addClass(SyncTargetFilesystem); SyncTargetRegistry.addClass(SyncTargetOneDrive); SyncTargetRegistry.addClass(SyncTargetNextcloud); SyncTargetRegistry.addClass(SyncTargetDropbox); +SyncTargetRegistry.addClass(SyncTargetAmazonS3); // const syncTargetId_ = SyncTargetRegistry.nameToId("nextcloud"); const syncTargetId_ = SyncTargetRegistry.nameToId('memory'); // const syncTargetId_ = SyncTargetRegistry.nameToId('filesystem'); // const syncTargetId_ = SyncTargetRegistry.nameToId('dropbox'); +// const syncTargetId_ = SyncTargetRegistry.nameToId('amazon_s3'); const syncDir = `${__dirname}/../tests/sync`; const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 100;// 400; @@ -351,8 +356,15 @@ function fileApi() { if (!authToken) throw new Error(`Dropbox auth token missing in ${authTokenPath}`); api.setAuthToken(authToken); fileApi_ = new FileApi('', new FileApiDriverDropbox(api)); + } else if (syncTargetId_ == SyncTargetRegistry.nameToId('amazon_s3')) { + const amazonS3CredsPath = `${__dirname}/support/amazon-s3-auth.json`; + const amazonS3Creds = require(amazonS3CredsPath); + if (!amazonS3Creds || !amazonS3Creds.accessKeyId) throw new Error(`AWS auth JSON missing in ${amazonS3CredsPath} format should be: { "accessKeyId": "", "secretAccessKey": "", "bucket": "mybucket"}`); + const api = new S3({ accessKeyId: amazonS3Creds.accessKeyId, secretAccessKey: amazonS3Creds.secretAccessKey, s3UseArnRegion: true }); + fileApi_ = new FileApi('', new FileApiDriverAmazonS3(api, amazonS3Creds.bucket)); } + fileApi_.setLogger(logger); fileApi_.setSyncTargetId(syncTargetId_); fileApi_.requestRepeatCount_ = 0; diff --git a/ElectronClient/package-lock.json b/ElectronClient/package-lock.json index fe364224a..5407a6b64 100644 --- a/ElectronClient/package-lock.json +++ b/ElectronClient/package-lock.json @@ -715,6 +715,34 @@ "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, + "aws-sdk": { + "version": "2.680.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.680.0.tgz", + "integrity": "sha512-sq19d5cNrgtcoMQc8GlwRrN11zT5FVxc+ZHL9P6lNAlGA3av3dwpt6+4smvhHpPzpzT0fG5A7HMczgjbLaLUDA==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -1946,6 +1974,15 @@ "readable-stream": "^3.4.0" }, "dependencies": { + "buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -2132,12 +2169,13 @@ "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" }, "buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "requires": { "base64-js": "^1.0.2", - "ieee754": "^1.1.4" + "ieee754": "^1.1.4", + "isarray": "^1.0.0" } }, "buffer-crc32": { @@ -4334,6 +4372,11 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -6852,6 +6895,11 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "joplin-turndown": { "version": "4.0.28", "resolved": "https://registry.npmjs.org/joplin-turndown/-/joplin-turndown-4.0.28.tgz", @@ -9506,6 +9554,11 @@ "strict-uri-encode": "^1.0.0" } }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "querystringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.0.tgz", @@ -11660,6 +11713,22 @@ "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=" }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, "url-parse": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.4.tgz", diff --git a/ElectronClient/package.json b/ElectronClient/package.json index 39d8f8cbb..f2abc68f2 100644 --- a/ElectronClient/package.json +++ b/ElectronClient/package.json @@ -99,6 +99,7 @@ "@fortawesome/fontawesome-free": "^5.13.0", "app-module-path": "^2.2.0", "async-mutex": "^0.1.3", + "aws-sdk": "^2.594.0", "base-64": "^0.1.0", "base64-stream": "^1.0.0", "chokidar": "^3.0.0", diff --git a/ReactNativeClient/lib/BaseApplication.js b/ReactNativeClient/lib/BaseApplication.js index 76018730c..1ee8476fc 100644 --- a/ReactNativeClient/lib/BaseApplication.js +++ b/ReactNativeClient/lib/BaseApplication.js @@ -29,6 +29,7 @@ const SyncTargetOneDriveDev = require('lib/SyncTargetOneDriveDev.js'); const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js'); const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js'); const SyncTargetDropbox = require('lib/SyncTargetDropbox.js'); +const SyncTargetAmazonS3 = require('lib/SyncTargetAmazonS3.js'); const EncryptionService = require('lib/services/EncryptionService'); const ResourceFetcher = require('lib/services/ResourceFetcher'); const SearchEngineUtils = require('lib/services/SearchEngineUtils'); @@ -629,6 +630,7 @@ class BaseApplication { SyncTargetRegistry.addClass(SyncTargetNextcloud); SyncTargetRegistry.addClass(SyncTargetWebDAV); SyncTargetRegistry.addClass(SyncTargetDropbox); + SyncTargetRegistry.addClass(SyncTargetAmazonS3); await shim.fsDriver().remove(tempDir); diff --git a/ReactNativeClient/lib/SyncTargetAmazonS3.js b/ReactNativeClient/lib/SyncTargetAmazonS3.js new file mode 100644 index 000000000..a11af3138 --- /dev/null +++ b/ReactNativeClient/lib/SyncTargetAmazonS3.js @@ -0,0 +1,110 @@ +const BaseSyncTarget = require('lib/BaseSyncTarget.js'); +const { _ } = require('lib/locale.js'); +const Setting = require('lib/models/Setting.js'); +const { FileApi } = require('lib/file-api.js'); +const { Synchronizer } = require('lib/synchronizer.js'); +const { FileApiDriverAmazonS3 } = require('lib/file-api-driver-amazon-s3.js'); +const S3 = require('aws-sdk/clients/s3'); + +class SyncTargetAmazonS3 extends BaseSyncTarget { + static id() { + return 8; + } + + static supportsConfigCheck() { + return true; + } + + constructor(db, options = null) { + super(db, options); + this.api_ = null; + } + + static targetName() { + return 'amazon_s3'; + } + + static label() { + return _('AWS S3') + ' (Beta)'; + } + + async isAuthenticated() { + return true; + } + + static s3BucketName() { + return Setting.value('sync.8.path'); + } + + s3AuthParameters() { + return { + accessKeyId: Setting.value('sync.8.username'), + secretAccessKey: Setting.value('sync.8.password'), + s3UseArnRegion: true, // override the request region with the region inferred from requested resource's ARN + }; + } + + api() { + if (this.api_) return this.api_; + + this.api_ = new S3(this.s3AuthParameters()); + return this.api_; + } + + static async newFileApi_(syncTargetId, options) { + const apiOptions = { + accessKeyId: options.username(), + secretAccessKey: options.password(), + s3UseArnRegion: true, + }; + + const api = new S3(apiOptions); + const driver = new FileApiDriverAmazonS3(api, SyncTargetAmazonS3.s3BucketName()); + const fileApi = new FileApi('', driver); + fileApi.setSyncTargetId(syncTargetId); + return fileApi; + } + + static async checkConfig(options) { + const fileApi = await SyncTargetAmazonS3.newFileApi_(SyncTargetAmazonS3.id(), options); + fileApi.requestRepeatCount_ = 0; + + const output = { + ok: false, + errorMessage: '', + }; + + try { + const headBucketReq = new Promise((resolve, reject) => { + fileApi.driver().api().headBucket({ + Bucket: options.path(), + },(err, response) => { + if (err) reject(err); + else resolve(response); + }); + }); + const result = await headBucketReq; + if (!result) throw new Error(`AWS S3 bucket not found: ${SyncTargetAmazonS3.s3BucketName()}`); + output.ok = true; + } catch (error) { + output.errorMessage = error.message; + if (error.code) output.errorMessage += ` (Code ${error.code})`; + } + + return output; + } + + async initFileApi() { + const appDir = ''; + const fileApi = new FileApi(appDir, new FileApiDriverAmazonS3(this.api(), SyncTargetAmazonS3.s3BucketName())); + fileApi.setSyncTargetId(SyncTargetAmazonS3.id()); + + return fileApi; + } + + async initSynchronizer() { + return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType')); + } +} + +module.exports = SyncTargetAmazonS3; diff --git a/ReactNativeClient/lib/file-api-driver-amazon-s3.js b/ReactNativeClient/lib/file-api-driver-amazon-s3.js new file mode 100644 index 000000000..7348fd579 --- /dev/null +++ b/ReactNativeClient/lib/file-api-driver-amazon-s3.js @@ -0,0 +1,361 @@ +const { basicDelta } = require('lib/file-api'); +const { basename } = require('lib/path-utils'); +const { shim } = require('lib/shim'); +const JoplinError = require('lib/JoplinError'); + +const S3_MAX_DELETES = 1000; + +class FileApiDriverAmazonS3 { + constructor(api, s3_bucket) { + this.s3_bucket_ = s3_bucket; + this.api_ = api; + } + + api() { + return this.api_; + } + + requestRepeatCount() { + return 3; + } + + makePath_(path) { + if (!path) return ''; + return path; + } + + hasErrorCode_(error, errorCode) { + if (!error || typeof error.code !== 'string') return false; + return error.code.indexOf(errorCode) >= 0; + } + + // Need to make a custom promise, built-in promise is broken: https://github.com/aws/aws-sdk-js/issues/1436 + async s3GetObject(key) { + return new Promise((resolve, reject) => { + this.api().getObject({ + Bucket: this.s3_bucket_, + Key: key, + }, (err, response) => { + if (err) reject(err); + else resolve(response); + }); + }); + } + + async s3ListObjects(key, cursor) { + return new Promise((resolve, reject) => { + this.api().listObjectsV2({ + Bucket: this.s3_bucket_, + Prefix: key, + Delimiter: '/', + ContinuationToken: cursor, + }, (err, response) => { + if (err) reject(err); + else resolve(response); + }); + }); + } + + async s3HeadObject(key) { + return new Promise((resolve, reject) => { + this.api().headObject({ + Bucket: this.s3_bucket_, + Key: key, + }, (err, response) => { + if (err) reject(err); + else resolve(response); + }); + }); + } + + async s3PutObject(key, body) { + return new Promise((resolve, reject) => { + this.api().putObject({ + Bucket: this.s3_bucket_, + Key: key, + Body: body, + }, (err, response) => { + if (err) reject(err); + else resolve(response); + }); + }); + } + + async s3UploadFileFrom(path, key) { + if (!shim.fsDriver().exists(path)) throw new Error('s3UploadFileFrom: file does not exist'); + const body = await shim.fsDriver().readFile(path, 'Buffer'); + return new Promise((resolve, reject) => { + this.api().upload({ + Bucket: this.s3_bucket_, + Key: key, + Body: body, + }, (err, response) => { + if (err) reject(err); + else resolve(response); + }); + }); + } + + async s3DeleteObject(key) { + return new Promise((resolve, reject) => { + this.api().deleteObject({ + Bucket: this.s3_bucket_, + Key: key, + }, + (err, response) => { + if (err) { + console.log(err.code); + console.log(err.message); + reject(err); + } else { resolve(response); } + }); + }); + } + + // Assumes key is formatted, like `{Key: 's3 path'}` + async s3DeleteObjects(keys) { + return new Promise((resolve, reject) => { + this.api().deleteObjects({ + Bucket: this.s3_bucket_, + Delete: { Objects: keys }, + }, + (err, response) => { + if (err) { + console.log(err.code); + console.log(err.message); + reject(err); + } else { resolve(response); } + }); + }); + } + + async stat(path) { + try { + const metadata = await this.s3HeadObject(this.makePath_(path)); + + return this.metadataToStat_(metadata, path); + } catch (error) { + if (this.hasErrorCode_(error, 'NotFound')) { + // ignore + } else { + throw error; + } + } + } + + metadataToStat_(md, path) { + const relativePath = basename(path); + + const output = { + path: relativePath, + updated_time: md['LastModified'] ? new Date(md['LastModified']) : new Date(), + isDeleted: !!md['DeleteMarker'], + isDir: false, + }; + + return output; + } + + metadataToStats_(mds) { + const output = []; + for (let i = 0; i < mds.length; i++) { + output.push(this.metadataToStat_(mds[i], mds[i].Key)); + } + return output; + } + + async setTimestamp() { + throw new Error('Not implemented'); // Not needed anymore + } + + async delta(path, options) { + const getDirStats = async path => { + const result = await this.list(path); + return result.items; + }; + + return await basicDelta(path, getDirStats, options); + } + + async list(path) { + let prefixPath = this.makePath_(path); + const pathLen = prefixPath.length; + if (pathLen > 0 && prefixPath[pathLen - 1] !== '/') { + prefixPath = `${prefixPath}/`; + } + + let response = await this.s3ListObjects(prefixPath); + + let output = this.metadataToStats_(response.Contents, prefixPath); + + while (response.IsTruncated) { + response = await this.s3ListObjects(prefixPath, response.NextContinuationToken); + + output = output.concat(this.metadataToStats_(response.Contents, prefixPath)); + } + + return { + items: output, + hasMore: false, + context: { cursor: response.NextContinuationToken }, + }; + } + + async get(path, options) { + const remotePath = this.makePath_(path); + if (!options) options = {}; + const responseFormat = options.responseFormat || 'text'; + + try { + let output = null; + const response = await this.s3GetObject(remotePath); + output = response.Body; + + if (options.target === 'file') { + const filePath = options.path; + if (!filePath) throw new Error('get: target options.path is missing'); + + // TODO: check if this ever hits on RN + await shim.fsDriver().writeBinaryFile(filePath, output); + return { + ok: true, + path: filePath, + text: () => { + return response.statusMessage; + }, + json: () => { + return { message: `${response.statusCode}: ${response.statusMessage}` }; + }, + status: response.statusCode, + headers: response.headers, + }; + } + + if (responseFormat === 'text') { + output = output.toString(); + } + + return output; + } catch (error) { + if (this.hasErrorCode_(error, 'NoSuchKey')) { + return null; + } else if (this.hasErrorCode_(error, 'AccessDenied')) { + throw new JoplinError('Do not have proper permissions to Bucket', 'rejectedByTarget'); + } else { + throw error; + } + } + } + + // Don't need to make directories, S3 is key based storage. + async mkdir() { + return true; + } + + async put(path, content, options = null) { + const remotePath = this.makePath_(path); + if (!options) options = {}; + + // See https://github.com/facebook/react-native/issues/14445#issuecomment-352965210 + if (typeof content === 'string') content = shim.Buffer.from(content, 'utf8'); + + try { + if (options.source === 'file') { + await this.s3UploadFileFrom(options.path, remotePath); + return; + } + + await this.s3PutObject(remotePath, content); + } catch (error) { + if (this.hasErrorCode_(error, 'AccessDenied')) { + throw new JoplinError('Do not have proper permissions to Bucket', 'rejectedByTarget'); + } else { + throw error; + } + } + } + + async delete(path) { + try { + await this.s3DeleteObject(this.makePath_(path)); + } catch (error) { + if (this.hasErrorCode_(error, 'NoSuchKey')) { + // ignore + } else { + throw error; + } + } + } + + async batchDeletes(paths) { + const keys = paths.map(path => { return { Key: path }; }); + while (keys.length > 0) { + const toDelete = keys.splice(0, S3_MAX_DELETES); + + try { + await this.s3DeleteObjects(toDelete); + } catch (error) { + if (this.hasErrorCode_(error, 'NoSuchKey')) { + // ignore + } else { + throw error; + } + } + } + } + + async move(oldPath, newPath) { + const req = new Promise((resolve, reject) => { + this.api().copyObject({ + Bucket: this.s3_bucket_, + CopySource: this.makePath_(oldPath), + Key: newPath, + },(err, response) => { + if (err) reject(err); + else resolve(response); + }); + }); + + try { + await req; + + this.delete(oldPath); + } catch (error) { + if (this.hasErrorCode_(error, 'NoSuchKey')) { + // ignore + } else { + throw error; + } + } + } + + format() { + throw new Error('Not supported'); + } + + async clearRoot() { + const listRecursive = async (cursor) => { + return new Promise((resolve, reject) => { + return this.api().listObjectsV2({ + Bucket: this.s3_bucket_, + ContinuationToken: cursor, + }, (err, response) => { + if (err) reject(err); + else resolve(response); + }); + }); + }; + + let response = await listRecursive(); + let keys = response.Contents.map((content) => content.Key); + + while (response.IsTruncated) { + response = await listRecursive(response.NextContinuationToken); + keys = keys.concat(response.Contents.map((content) => content.Key)); + } + + this.batchDeletes(keys); + } +} + +module.exports = { FileApiDriverAmazonS3 }; diff --git a/ReactNativeClient/lib/file-api.js b/ReactNativeClient/lib/file-api.js index 754960090..bf12d318c 100644 --- a/ReactNativeClient/lib/file-api.js +++ b/ReactNativeClient/lib/file-api.js @@ -172,6 +172,7 @@ class FileApi { // }); } + // Returns UTF-8 encoded string by default, or a Response if `options.target = 'file'` get(path, options = null) { if (!options) options = {}; if (!options.encoding) options.encoding = 'utf8'; diff --git a/ReactNativeClient/lib/models/Setting.js b/ReactNativeClient/lib/models/Setting.js index b762a2398..f1fb7adce 100644 --- a/ReactNativeClient/lib/models/Setting.js +++ b/ReactNativeClient/lib/models/Setting.js @@ -171,16 +171,45 @@ class Setting extends BaseModel { secure: true, }, - 'sync.3.auth': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.4.auth': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.7.auth': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.1.context': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.2.context': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.3.context': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.4.context': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.5.context': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.6.context': { value: '', type: Setting.TYPE_STRING, public: false }, - 'sync.7.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.8.path': { + value: '', + type: Setting.TYPE_STRING, + section: 'sync', + show: settings => { + try { + return settings['sync.target'] == SyncTargetRegistry.nameToId('amazon_s3'); + } catch (error) { + return false; + } + }, + filter: value => { + return value ? rtrimSlashes(value) : ''; + }, + public: true, + label: () => _('AWS S3 bucket'), + description: () => emptyDirWarning, + }, + 'sync.8.username': { + value: '', + type: Setting.TYPE_STRING, + section: 'sync', + show: settings => { + return settings['sync.target'] == SyncTargetRegistry.nameToId('amazon_s3'); + }, + public: true, + label: () => _('AWS key'), + }, + 'sync.8.password': { + value: '', + type: Setting.TYPE_STRING, + section: 'sync', + show: settings => { + return settings['sync.target'] == SyncTargetRegistry.nameToId('amazon_s3'); + }, + public: true, + label: () => _('AWS secret'), + secure: true, + }, 'sync.5.syncTargets': { value: {}, type: Setting.TYPE_OBJECT, public: false }, @@ -203,6 +232,18 @@ class Setting extends BaseModel { }, }, + 'sync.3.auth': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.4.auth': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.7.auth': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.1.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.2.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.3.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.4.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.5.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.6.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.7.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.8.context': { value: '', type: Setting.TYPE_STRING, public: false }, + 'sync.maxConcurrentConnections': { value: 5, type: Setting.TYPE_INT, public: true, advanced: true, section: 'sync', label: () => _('Max concurrent connections'), minimum: 1, maximum: 20, step: 1 }, activeFolderId: { value: '', type: Setting.TYPE_STRING, public: false }, diff --git a/ReactNativeClient/package-lock.json b/ReactNativeClient/package-lock.json index ae1bb3998..6b34cda5f 100644 --- a/ReactNativeClient/package-lock.json +++ b/ReactNativeClient/package-lock.json @@ -2533,6 +2533,63 @@ "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, + "aws-sdk": { + "version": "2.642.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.642.0.tgz", + "integrity": "sha512-0ZNgL1HBXRVobFD9Z64RyQk50cNABDMU1GV4lYIAvao4urYqYJi2MEVQmq+7WyXyzkBWu3lAPNDiJ8WW7emTzg==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "babel-code-frame": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", @@ -6246,6 +6303,11 @@ "resolved": "https://registry.npmjs.org/jetifier/-/jetifier-1.6.5.tgz", "integrity": "sha512-T7yzBSu9PR+DqjYt+I0KVO1XTb1QhAfHnXV5Nd3xpbXM6Xg4e3vP60Q4qkNU8Fh6PHC2PivPUNN3rY7G2MxcDQ==" }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "js-tokens": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", diff --git a/ReactNativeClient/package.json b/ReactNativeClient/package.json index e590415c7..bc3dff21f 100644 --- a/ReactNativeClient/package.json +++ b/ReactNativeClient/package.json @@ -18,6 +18,7 @@ "@react-native-community/push-notification-ios": "^1.0.5", "@react-native-community/slider": "^2.0.8", "async-mutex": "^0.1.3", + "aws-sdk": "^2.588.0", "base-64": "^0.1.0", "buffer": "^5.0.8", "color": "^3.1.2",