const { basicDelta } = require('./file-api'); const { basename } = require('./path-utils'); const shim = require('./shim').default; const JoplinError = require('./JoplinError').default; const { Buffer } = require('buffer'); const { GetObjectCommand, ListObjectsV2Command, HeadObjectCommand, PutObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, CopyObjectCommand } = require('@aws-sdk/client-s3'); const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); const parser = require('fast-xml-parser'); const S3_MAX_DELETES = 1000; class FileApiDriverAmazonS3 { constructor(api, s3_bucket) { this.s3_bucket_ = s3_bucket; this.api_ = api; } api() { return this.api_; } requestRepeatCount() { return 3; } makePath_(path) { if (!path) return ''; return path; } hasErrorCode_(error, errorCode) { if (!error) return false; if (error.name) { return error.name.indexOf(errorCode) >= 0; } else if (error.code) { return error.code.indexOf(errorCode) >= 0; } else if (error.Code) { return error.Code.indexOf(errorCode) >= 0; } else { return false; } } // Because of the way AWS-SDK-v3 works for getting data from a bucket we will // use a pre-signed URL to avoid https://github.com/aws/aws-sdk-js-v3/issues/1877 async s3GenerateGetURL(key) { const signedUrl = await getSignedUrl(this.api(), new GetObjectCommand({ Bucket: this.s3_bucket_, Key: key, }), { expiresIn: 3600, }); return signedUrl; } // We've now moved to aws-sdk-v3 and this note is outdated, but explains the promise structure. // Need to make a custom promise, built-in promise is broken: https://github.com/aws/aws-sdk-js/issues/1436 // TODO: Re-factor to https://github.com/aws/aws-sdk-js-v3/tree/main/clients/client-s3#asyncawait async s3ListObjects(key, cursor) { return new Promise((resolve, reject) => { this.api().send(new ListObjectsV2Command({ Bucket: this.s3_bucket_, Prefix: key, Delimiter: '/', ContinuationToken: cursor, }), (err, response) => { if (err) reject(err); else resolve(response); }); }); } async s3HeadObject(key) { return new Promise((resolve, reject) => { this.api().send(new HeadObjectCommand({ Bucket: this.s3_bucket_, Key: key, }), (err, response) => { if (err) reject(err); else resolve(response); }); }); } async s3PutObject(key, body) { return new Promise((resolve, reject) => { this.api().send(new PutObjectCommand({ Bucket: this.s3_bucket_, Key: key, Body: body, }), (err, response) => { if (err) reject(err); else resolve(response); }); }); } async s3UploadFileFrom(path, key) { if (!shim.fsDriver().exists(path)) throw new Error('s3UploadFileFrom: file does not exist'); const body = await shim.fsDriver().readFile(path, 'base64'); const fileStat = await shim.fsDriver().stat(path); return new Promise((resolve, reject) => { this.api().send(new PutObjectCommand({ Bucket: this.s3_bucket_, Key: key, Body: Buffer.from(body, 'base64'), ContentLength: `${fileStat.size}`, }), (err, response) => { if (err) reject(err); else resolve(response); }); }); } async s3DeleteObject(key) { return new Promise((resolve, reject) => { this.api().send(new DeleteObjectCommand({ Bucket: this.s3_bucket_, Key: key, }), (err, response) => { if (err) { console.log(err.code); console.log(err.message); reject(err); } else { resolve(response); } }); }); } // Assumes key is formatted, like `{Key: 's3 path'}` async s3DeleteObjects(keys) { return new Promise((resolve, reject) => { this.api().send(new DeleteObjectsCommand({ Bucket: this.s3_bucket_, Delete: { Objects: keys }, }), (err, response) => { if (err) { console.log(err.code); console.log(err.message); reject(err); } else { resolve(response); } }); }); } async stat(path) { try { const metadata = await this.s3HeadObject(this.makePath_(path)); return this.metadataToStat_(metadata, path); } catch (error) { if (this.hasErrorCode_(error, 'NotFound')) { // ignore } else { throw error; } } } metadataToStat_(md, path) { const relativePath = basename(path); const lastModifiedDate = md['LastModified'] ? new Date(md['LastModified']) : new Date(); const output = { path: relativePath, updated_time: lastModifiedDate.getTime(), isDeleted: !!md['DeleteMarker'], isDir: false, }; return output; } metadataToStats_(mds) { const output = []; for (let i = 0; i < mds.length; i++) { output.push(this.metadataToStat_(mds[i], mds[i].Key)); } return output; } async setTimestamp() { throw new Error('Not implemented'); // Not needed anymore } async delta(path, options) { const getDirStats = async path => { const result = await this.list(path); return result.items; }; return await basicDelta(path, getDirStats, options); } async list(path) { let prefixPath = this.makePath_(path); const pathLen = prefixPath.length; if (pathLen > 0 && prefixPath[pathLen - 1] !== '/') { prefixPath = `${prefixPath}/`; } // There is a bug/quirk of aws-sdk-js-v3 which causes the // S3Client systemClockOffset to be wildly inaccurate. This // effectively removes the offset and sets it to system time. // See https://github.com/aws/aws-sdk-js-v3/issues/2208 for more. // If the user's time actaully off, then this should correctly // result in a RequestTimeTooSkewed error from s3ListObjects. this.api().config.systemClockOffset = 0; let response = await this.s3ListObjects(prefixPath); // In aws-sdk-js-v3 if there are no contents it no longer returns // an empty array. This creates an Empty array to pass onward. if (response.Contents === undefined) response.Contents = []; let output = this.metadataToStats_(response.Contents, prefixPath); while (response.IsTruncated) { response = await this.s3ListObjects(prefixPath, response.NextContinuationToken); output = output.concat(this.metadataToStats_(response.Contents, prefixPath)); } return { items: output, hasMore: false, context: { cursor: response.NextContinuationToken }, }; } async get(path, options) { const remotePath = this.makePath_(path); if (!options) options = {}; const responseFormat = options.responseFormat || 'text'; try { let output = null; let response = null; const s3Url = await this.s3GenerateGetURL(remotePath); if (options.target === 'file') { output = await shim.fetchBlob(s3Url, options); } else if (responseFormat === 'text') { response = await shim.fetch(s3Url, options); output = await response.text(); // we need to make sure that errors get thrown as we are manually fetching above. if (!response.ok) { throw { name: response.statusText, output: output }; } } return output; } catch (error) { // This means that the error was on the Desktop client side and we need to handle that. // On Mobile it won't match because FetchError is a node-fetch feature. // https://github.com/node-fetch/node-fetch/blob/main/docs/ERROR-HANDLING.md if (error.name === 'FetchError') { throw error.message; } let parsedOutput = ''; // If error.output is not xml the last else case should // actually let us see the output of error. if (error.output) { parsedOutput = parser.parse(error.output); if (this.hasErrorCode_(parsedOutput.Error, 'AuthorizationHeaderMalformed')) { throw error.output; } if (this.hasErrorCode_(parsedOutput.Error, 'NoSuchKey')) { return null; } else if (this.hasErrorCode_(parsedOutput.Error, 'AccessDenied')) { throw new JoplinError('Do not have proper permissions to Bucket', 'rejectedByTarget'); } } else { if (error.output) { throw error.output; } else { throw error; } } } } // Don't need to make directories, S3 is key based storage. async mkdir() { return true; } async put(path, content, options = null) { const remotePath = this.makePath_(path); if (!options) options = {}; // See https://github.com/facebook/react-native/issues/14445#issuecomment-352965210 if (typeof content === 'string') content = shim.Buffer.from(content, 'utf8'); try { if (options.source === 'file') { await this.s3UploadFileFrom(options.path, remotePath); return; } await this.s3PutObject(remotePath, content); } catch (error) { if (this.hasErrorCode_(error, 'AccessDenied')) { throw new JoplinError('Do not have proper permissions to Bucket', 'rejectedByTarget'); } else { throw error; } } } async delete(path) { try { await this.s3DeleteObject(this.makePath_(path)); } catch (error) { if (this.hasErrorCode_(error, 'NoSuchKey')) { // ignore } else { throw error; } } } async batchDeletes(paths) { const keys = paths.map(path => { return { Key: path }; }); while (keys.length > 0) { const toDelete = keys.splice(0, S3_MAX_DELETES); try { await this.s3DeleteObjects(toDelete); } catch (error) { if (this.hasErrorCode_(error, 'NoSuchKey')) { // ignore } else { throw error; } } } } async move(oldPath, newPath) { const req = new Promise((resolve, reject) => { this.api().send(new CopyObjectCommand({ Bucket: this.s3_bucket_, CopySource: this.makePath_(oldPath), Key: newPath, }), (err, response) => { if (err) reject(err); else resolve(response); }); }); try { await req; this.delete(oldPath); } catch (error) { if (this.hasErrorCode_(error, 'NoSuchKey')) { // ignore } else { throw error; } } } format() { throw new Error('Not supported'); } async clearRoot() { const listRecursive = async (cursor) => { return new Promise((resolve, reject) => { return this.api().send(new ListObjectsV2Command({ Bucket: this.s3_bucket_, ContinuationToken: cursor, }), (err, response) => { if (err) reject(err); else resolve(response); }); }); }; let response = await listRecursive(); // In aws-sdk-js-v3 if there are no contents it no longer returns // an empty array. This creates an Empty array to pass onward. if (response.Contents === undefined) response.Contents = []; let keys = response.Contents.map((content) => content.Key); while (response.IsTruncated) { response = await listRecursive(response.NextContinuationToken); keys = keys.concat(response.Contents.map((content) => content.Key)); } this.batchDeletes(keys); } } module.exports = { FileApiDriverAmazonS3 };