From 57731b6cd261c732041aacc094429ab02ca13418 Mon Sep 17 00:00:00 2001 From: udf2457 Date: Wed, 17 Apr 2024 02:58:13 +0100 Subject: [PATCH] S3 SSE-C encryption support. This feature (enabled with --repo-s3-sse-customer-key) provides an encryption key to encrypt the data after it has been transmitted to the server. While not as secure as encrypting data before transmission (--repo-cipher-type), this may be useful in certain configurations. --- doc/xml/release/2024/2.52.xml | 15 ++++++ doc/xml/release/contributor.xml | 5 ++ src/build/config/config.yaml | 5 ++ src/build/help/help.xml | 10 ++++ src/config/config.auto.h | 3 +- src/config/parse.auto.c.inc | 82 ++++++++++++++++++++++++++++++ src/storage/s3/helper.c | 5 +- src/storage/s3/read.c | 2 +- src/storage/s3/storage.c | 37 ++++++++++++-- src/storage/s3/storage.h | 6 +-- src/storage/s3/storage.intern.h | 2 + src/storage/s3/write.c | 6 +-- test/src/common/harnessHost.c | 4 +- test/src/module/command/helpTest.c | 1 + test/src/module/storage/s3Test.c | 62 +++++++++++++++++----- 15 files changed, 215 insertions(+), 30 deletions(-) diff --git a/doc/xml/release/2024/2.52.xml b/doc/xml/release/2024/2.52.xml index d8175f0a1..b2c2bdc41 100644 --- a/doc/xml/release/2024/2.52.xml +++ b/doc/xml/release/2024/2.52.xml @@ -1,5 +1,20 @@ + + + + + + + + + + + +

S3 SSE-C encryption support.

+ + + diff --git a/doc/xml/release/contributor.xml b/doc/xml/release/contributor.xml index 793648e9a..c8b75a84e 100644 --- a/doc/xml/release/contributor.xml +++ b/doc/xml/release/contributor.xml @@ -1020,6 +1020,11 @@ ucando + + udf2457 + udf2457 + + Ugo Bellavance diff --git a/src/build/config/config.yaml b/src/build/config/config.yaml index a8ff09b30..bd3fbac8f 100644 --- a/src/build/config/config.yaml +++ b/src/build/config/config.yaml @@ -2333,6 +2333,11 @@ option: inherit: repo-s3-bucket required: false + repo-s3-sse-customer-key: + inherit: repo-s3-bucket + required: false + secure: true + repo-s3-region: inherit: repo-s3-bucket deprecate: diff --git a/src/build/help/help.xml b/src/build/help/help.xml index af8aa1b21..5c87fa441 100644 --- a/src/build/help/help.xml +++ b/src/build/help/help.xml @@ -1044,6 +1044,16 @@ us-east-1 + + S3 Repository SSE Customer Key. + + +

Setting this option enables S3 server-side encryption using the specified customer key.

+ + + bceb4f13-6939-4be3-910d-df54dee817b7 + + S3 URI Style. diff --git a/src/config/config.auto.h b/src/config/config.auto.h index cb9204c01..cda6f3869 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -136,7 +136,7 @@ Option constants #define CFGOPT_TYPE "type" #define CFGOPT_VERBOSE "verbose" -#define CFG_OPTION_TOTAL 179 +#define CFG_OPTION_TOTAL 180 /*********************************************************************************************************************************** Option value constants @@ -516,6 +516,7 @@ typedef enum cfgOptRepoS3KmsKeyId, cfgOptRepoS3Region, cfgOptRepoS3Role, + cfgOptRepoS3SseCustomerKey, cfgOptRepoS3Token, cfgOptRepoS3UriStyle, cfgOptRepoSftpHost, diff --git a/src/config/parse.auto.c.inc b/src/config/parse.auto.c.inc index 609baa1c2..f20b353ed 100644 --- a/src/config/parse.auto.c.inc +++ b/src/config/parse.auto.c.inc @@ -7723,6 +7723,87 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = ), // opt/repo-s3-role ), // opt/repo-s3-role // ----------------------------------------------------------------------------------------------------------------------------- + PARSE_RULE_OPTION // opt/repo-s3-sse-customer-key + ( // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_NAME("repo-s3-sse-customer-key"), // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_TYPE(cfgOptTypeString), // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_RESET(true), // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_REQUIRED(false), // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_SECURE(true), // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_GROUP_MEMBER(true), // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo), // opt/repo-s3-sse-customer-key + // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-s3-sse-customer-key + ( // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdAnnotate) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdExpire) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdManifest) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-s3-sse-customer-key + ), // opt/repo-s3-sse-customer-key + // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/repo-s3-sse-customer-key + ( // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-s3-sse-customer-key + ), // opt/repo-s3-sse-customer-key + // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/repo-s3-sse-customer-key + ( // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-s3-sse-customer-key + ), // opt/repo-s3-sse-customer-key + // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/repo-s3-sse-customer-key + ( // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdAnnotate) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdManifest) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-s3-sse-customer-key + ), // opt/repo-s3-sse-customer-key + // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTIONAL // opt/repo-s3-sse-customer-key + ( // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTIONAL_GROUP // opt/repo-s3-sse-customer-key + ( // opt/repo-s3-sse-customer-key + PARSE_RULE_OPTIONAL_DEPEND // opt/repo-s3-sse-customer-key + ( // opt/repo-s3-sse-customer-key + PARSE_RULE_VAL_OPT(cfgOptRepoType), // opt/repo-s3-sse-customer-key + PARSE_RULE_VAL_STRID(parseRuleValStrIdS3), // opt/repo-s3-sse-customer-key + ), // opt/repo-s3-sse-customer-key + ), // opt/repo-s3-sse-customer-key + ), // opt/repo-s3-sse-customer-key + ), // opt/repo-s3-sse-customer-key + // ----------------------------------------------------------------------------------------------------------------------------- PARSE_RULE_OPTION // opt/repo-s3-token ( // opt/repo-s3-token PARSE_RULE_OPTION_NAME("repo-s3-token"), // opt/repo-s3-token @@ -10977,6 +11058,7 @@ static const uint8_t optionResolveOrder[] = cfgOptRepoS3KmsKeyId, // opt-resolve-order cfgOptRepoS3Region, // opt-resolve-order cfgOptRepoS3Role, // opt-resolve-order + cfgOptRepoS3SseCustomerKey, // opt-resolve-order cfgOptRepoS3Token, // opt-resolve-order cfgOptRepoS3UriStyle, // opt-resolve-order cfgOptRepoSftpHost, // opt-resolve-order diff --git a/src/storage/s3/helper.c b/src/storage/s3/helper.c index 6cc9ececa..42fe093b7 100644 --- a/src/storage/s3/helper.c +++ b/src/storage/s3/helper.c @@ -85,8 +85,9 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, - webIdToken, (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), + cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), + cfgOptionIdxStrNull(cfgOptRepoS3SseCustomerKey, repoIdx), role, webIdToken, + (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), cfgOptionIdxKvNull(cfgOptRepoStorageTag, repoIdx), host, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); diff --git a/src/storage/s3/read.c b/src/storage/s3/read.c index 9c3921eff..c02d7662a 100644 --- a/src/storage/s3/read.c +++ b/src/storage/s3/read.c @@ -55,7 +55,7 @@ storageReadS3Open(THIS_VOID) this->httpResponse = storageS3RequestP( this->storage, HTTP_VERB_GET_STR, this->interface.name, .header = httpHeaderPutRange(httpHeaderNew(NULL), this->interface.offset, this->interface.limit), - .allowMissing = true, .contentIo = true); + .allowMissing = true, .contentIo = true, .sseC = true); } MEM_CONTEXT_OBJ_END(); diff --git a/src/storage/s3/storage.c b/src/storage/s3/storage.c index c71374a2b..3a51e2c1e 100644 --- a/src/storage/s3/storage.c +++ b/src/storage/s3/storage.c @@ -33,6 +33,10 @@ STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-secur STRING_STATIC(S3_HEADER_SRVSDENC_STR, "x-amz-server-side-encryption"); STRING_STATIC(S3_HEADER_SRVSDENC_KMS_STR, "aws:kms"); STRING_STATIC(S3_HEADER_SRVSDENC_KMSKEYID_STR, "x-amz-server-side-encryption-aws-kms-key-id"); +STRING_STATIC(S3_HEADER_SSECUSTKEY_ALGO_STR, "x-amz-server-side-encryption-customer-algorithm"); +STRING_STATIC(S3_HEADER_SSECUSTKEY_AES256_STR, "AES256"); +STRING_STATIC(S3_HEADER_SSECUSTKEY_KEY_STR, "x-amz-server-side-encryption-customer-key"); +STRING_STATIC(S3_HEADER_SSECUSTKEY_KEY_MD5_STR, "x-amz-server-side-encryption-customer-key-md5"); STRING_STATIC(S3_HEADER_TAGGING, "x-amz-tagging"); /*********************************************************************************************************************************** @@ -94,6 +98,8 @@ struct StorageS3 String *secretAccessKey; // Secret access key String *securityToken; // Security token, if any const String *kmsKeyId; // Server-side encryption key + const String *sseCustomerKey; // Base64 of SSE-C encryption key + const String *sseCustomerKeyMd5; // Base64 of MD5 of SSE-C key size_t partSize; // Part size for multi-part upload const String *tag; // Tags to be applied to objects unsigned int deleteMax; // Maximum objects that can be deleted in one request @@ -455,6 +461,7 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S FUNCTION_LOG_PARAM(HTTP_QUERY, param.query); FUNCTION_LOG_PARAM(BUFFER, param.content); FUNCTION_LOG_PARAM(BOOL, param.sseKms); + FUNCTION_LOG_PARAM(BOOL, param.sseC); FUNCTION_LOG_PARAM(BOOL, param.tag); FUNCTION_LOG_END(); @@ -489,6 +496,14 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S httpHeaderPut(requestHeader, S3_HEADER_SRVSDENC_KMSKEYID_STR, this->kmsKeyId); } + // Set SSE-C headers when requested + if (param.sseC && this->sseCustomerKey != NULL) + { + httpHeaderPut(requestHeader, S3_HEADER_SSECUSTKEY_ALGO_STR, S3_HEADER_SSECUSTKEY_AES256_STR); + httpHeaderPut(requestHeader, S3_HEADER_SSECUSTKEY_KEY_STR, this->sseCustomerKey); + httpHeaderPut(requestHeader, S3_HEADER_SSECUSTKEY_KEY_MD5_STR, this->sseCustomerKeyMd5); + } + // Set tags when requested and available if (param.tag && this->tag != NULL) httpHeaderPut(requestHeader, S3_HEADER_TAGGING, this->tag); @@ -599,12 +614,13 @@ storageS3Request(StorageS3 *this, const String *verb, const String *path, Storag FUNCTION_LOG_PARAM(BOOL, param.allowMissing); FUNCTION_LOG_PARAM(BOOL, param.contentIo); FUNCTION_LOG_PARAM(BOOL, param.sseKms); + FUNCTION_LOG_PARAM(BOOL, param.sseC); FUNCTION_LOG_PARAM(BOOL, param.tag); FUNCTION_LOG_END(); HttpRequest *const request = storageS3RequestAsyncP( this, verb, path, .header = param.header, .query = param.query, .content = param.content, .sseKms = param.sseKms, - .tag = param.tag); + .sseC = param.sseC, .tag = param.tag); HttpResponse *const result = storageS3ResponseP( request, .allowMissing = param.allowMissing, .contentIo = param.contentIo); @@ -801,7 +817,7 @@ storageS3Info(THIS_VOID, const String *const file, const StorageInfoLevel level, ASSERT(file != NULL); // Attempt to get file info - HttpResponse *const httpResponse = storageS3RequestP(this, HTTP_VERB_HEAD_STR, file, .allowMissing = true); + HttpResponse *const httpResponse = storageS3RequestP(this, HTTP_VERB_HEAD_STR, file, .allowMissing = true, .sseC = true); // Does the file exist? StorageInfo result = {.level = level, .exists = httpResponseCodeOk(httpResponse)}; @@ -1105,9 +1121,9 @@ storageS3New( const String *const path, const bool write, StoragePathExpressionCallback pathExpressionFunction, const String *const bucket, const String *const endPoint, const StorageS3UriStyle uriStyle, const String *const region, const StorageS3KeyType keyType, const String *const accessKey, const String *const secretAccessKey, const String *const securityToken, - const String *const kmsKeyId, const String *const credRole, const String *const webIdToken, const size_t partSize, - const KeyValue *const tag, const String *host, const unsigned int port, const TimeMSec timeout, const bool verifyPeer, - const String *const caFile, const String *const caPath) + const String *const kmsKeyId, const String *sseCustomerKey, const String *const credRole, const String *const webIdToken, + const size_t partSize, const KeyValue *const tag, const String *host, const unsigned int port, const TimeMSec timeout, + const bool verifyPeer, const String *const caFile, const String *const caPath) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, path); @@ -1122,6 +1138,7 @@ storageS3New( FUNCTION_TEST_PARAM(STRING, secretAccessKey); FUNCTION_TEST_PARAM(STRING, securityToken); FUNCTION_TEST_PARAM(STRING, kmsKeyId); + FUNCTION_TEST_PARAM(STRING, sseCustomerKey); FUNCTION_TEST_PARAM(STRING, credRole); FUNCTION_TEST_PARAM(STRING, webIdToken); FUNCTION_LOG_PARAM(SIZE, partSize); @@ -1149,6 +1166,7 @@ storageS3New( .region = strDup(region), .keyType = keyType, .kmsKeyId = strDup(kmsKeyId), + .sseCustomerKey = strDup(sseCustomerKey), .partSize = partSize, .deleteMax = STORAGE_S3_DELETE_MAX, .uriStyle = uriStyle, @@ -1227,10 +1245,19 @@ storageS3New( } } + // Generate SSE customer key MD5 hash + if (this->sseCustomerKey != NULL) + { + this->sseCustomerKeyMd5 = strNewEncode( + encodingBase64, cryptoHashOne(hashTypeMd5, bufNewDecode(encodingBase64, this->sseCustomerKey))); + } + // Create list of redacted headers this->headerRedactList = strLstNew(); strLstAdd(this->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR); strLstAdd(this->headerRedactList, S3_HEADER_DATE_STR); + strLstAdd(this->headerRedactList, S3_HEADER_SSECUSTKEY_KEY_STR); + strLstAdd(this->headerRedactList, S3_HEADER_SSECUSTKEY_KEY_MD5_STR); strLstAdd(this->headerRedactList, S3_HEADER_TOKEN_STR); } OBJ_NEW_END(); diff --git a/src/storage/s3/storage.h b/src/storage/s3/storage.h index e1eea2311..3154c00a6 100644 --- a/src/storage/s3/storage.h +++ b/src/storage/s3/storage.h @@ -36,8 +36,8 @@ Constructors FN_EXTERN Storage *storageS3New( const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey, - const String *secretAccessKey, const String *securityToken, const String *kmsKeyId, const String *credRole, - const String *webIdToken, size_t partSize, const KeyValue *tag, const String *host, unsigned int port, TimeMSec timeout, - bool verifyPeer, const String *caFile, const String *caPath); + const String *secretAccessKey, const String *securityToken, const String *kmsKeyId, const String *sseCustomerKey, + const String *credRole, const String *webIdToken, size_t partSize, const KeyValue *tag, const String *host, unsigned int port, + TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath); #endif diff --git a/src/storage/s3/storage.intern.h b/src/storage/s3/storage.intern.h index c41eed658..24be5bce9 100644 --- a/src/storage/s3/storage.intern.h +++ b/src/storage/s3/storage.intern.h @@ -23,6 +23,7 @@ typedef struct StorageS3RequestAsyncParam const HttpQuery *query; // Query parameters const Buffer *content; // Request content bool sseKms; // Enable server-side encryption? + bool sseC; // Enable server-side encryption with customer-provided keys? bool tag; // Add tags when available? } StorageS3RequestAsyncParam; @@ -54,6 +55,7 @@ typedef struct StorageS3RequestParam bool allowMissing; // Allow missing files (caller can check response code) bool contentIo; // Is IoRead interface required to read content? bool sseKms; // Enable server-side encryption? + bool sseC; // Enable server-side encryption with customer-provided keys? bool tag; // Add tags when available? } StorageS3RequestParam; diff --git a/src/storage/s3/write.c b/src/storage/s3/write.c index 86cbc65fa..492d922fa 100644 --- a/src/storage/s3/write.c +++ b/src/storage/s3/write.c @@ -128,7 +128,7 @@ storageWriteS3PartAsync(StorageWriteS3 *this) storageS3RequestP( this->storage, HTTP_VERB_POST_STR, this->interface.name, .query = httpQueryAdd(httpQueryNewP(), S3_QUERY_UPLOADS_STR, EMPTY_STR), .sseKms = true, - .tag = true)))); + .sseC = true, .tag = true)))); // Store the upload id MEM_CONTEXT_OBJ_BEGIN(this) @@ -147,7 +147,7 @@ storageWriteS3PartAsync(StorageWriteS3 *this) MEM_CONTEXT_OBJ_BEGIN(this) { this->request = storageS3RequestAsyncP( - this->storage, HTTP_VERB_PUT_STR, this->interface.name, .query = query, .content = this->partBuffer); + this->storage, HTTP_VERB_PUT_STR, this->interface.name, .query = query, .content = this->partBuffer, .sseC = true); } MEM_CONTEXT_OBJ_END(); } @@ -256,7 +256,7 @@ storageWriteS3Close(THIS_VOID) { storageS3RequestP( this->storage, HTTP_VERB_PUT_STR, this->interface.name, .content = this->partBuffer, .sseKms = true, - .tag = true); + .sseC = true, .tag = true); } bufFree(this->partBuffer); diff --git a/test/src/common/harnessHost.c b/test/src/common/harnessHost.c index 0453a504c..daec30ee2 100644 --- a/test/src/common/harnessHost.c +++ b/test/src/common/harnessHost.c @@ -748,8 +748,8 @@ hrnHostConfig(HrnHost *const this) this->pub.repo1Storage = storageS3New( hrnHostRepo1Path(this), true, NULL, STRDEF(HRN_HOST_S3_BUCKET), STRDEF(HRN_HOST_S3_ENDPOINT), storageS3UriStyleHost, STR(HRN_HOST_S3_REGION), storageS3KeyTypeShared, STRDEF(HRN_HOST_S3_ACCESS_KEY), - STRDEF(HRN_HOST_S3_ACCESS_SECRET_KEY), NULL, NULL, NULL, NULL, 5 * 1024 * 1024, NULL, hrnHostIp(s3), - 443, ioTimeoutMs(), false, NULL, NULL); + STRDEF(HRN_HOST_S3_ACCESS_SECRET_KEY), NULL, NULL, NULL, NULL, NULL, 5 * 1024 * 1024, NULL, + hrnHostIp(s3), 443, ioTimeoutMs(), false, NULL, NULL); } MEM_CONTEXT_OBJ_END(); diff --git a/test/src/module/command/helpTest.c b/test/src/module/command/helpTest.c index f58cb5c59..4351fca27 100644 --- a/test/src/module/command/helpTest.c +++ b/test/src/module/command/helpTest.c @@ -321,6 +321,7 @@ testRun(void) " --repo-s3-kms-key-id S3 repository KMS key\n" " --repo-s3-region S3 repository region\n" " --repo-s3-role S3 repository role\n" + " --repo-s3-sse-customer-key S3 Repository SSE Customer Key\n" " --repo-s3-token S3 repository security token\n" " --repo-s3-uri-style S3 URI Style [default=host]\n" " --repo-sftp-host SFTP repository host\n" diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index 910762c64..9c98eb5ef 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -29,6 +29,7 @@ typedef struct TestRequestParam const char *securityToken; const char *range; const char *kms; + const char *sseC; const char *ttl; const char *token; const char *tag; @@ -82,6 +83,14 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes if (param.kms != NULL) strCatZ(request, ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id"); + if (param.sseC != NULL) + { + strCatZ( + request, + ";x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key" + ";x-amz-server-side-encryption-customer-key-md5"); + } + if (param.tag != NULL) strCatZ(request, ";x-amz-tagging"); @@ -136,6 +145,16 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes strCatFmt(request, "x-amz-server-side-encryption-aws-kms-key-id:%s\r\n", param.kms); } + // Add sseC key + if (param.sseC != NULL) + { + strCatZ(request, "x-amz-server-side-encryption-customer-algorithm:AES256\r\n"); + strCatFmt(request, "x-amz-server-side-encryption-customer-key:%s\r\n", param.sseC); + strCatFmt( + request, "x-amz-server-side-encryption-customer-key-md5:%s\r\n", + strZ(strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, bufNewDecode(encodingBase64, STR(param.sseC)))))); + } + // Add tags if (param.tag != NULL) strCatFmt(request, "x-amz-tagging:%s\r\n", param.tag); @@ -492,9 +511,11 @@ testRun(void) hrnCfgArgRaw(argList, cfgOptRepoS3Role, credRole); hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeAuto); hrnCfgArgRawZ(argList, cfgOptRepoS3KmsKeyId, "kmskey1"); + hrnCfgEnvRawZ(cfgOptRepoS3SseCustomerKey, "rA1P"); hrnCfgArgRawZ(argList, cfgOptRepoStorageTag, "Key1=Value1"); hrnCfgArgRawZ(argList, cfgOptRepoStorageTag, " Key 2= Value 2"); HRN_CFG_LOAD(cfgCmdArchivePush, argList); + hrnCfgEnvRemoveRaw(cfgOptRepoS3SseCustomerKey); s3 = storageRepoGet(0, true); driver = (StorageS3 *)storageDriver(s3); @@ -660,7 +681,7 @@ testRun(void) hrnServerScriptClose(auth); - testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .accessKey = "x", .securityToken = "z"); + testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .accessKey = "x", .securityToken = "z", .sseC = "rA1P"); testResponseP(service, .code = 303, .content = "CONTENT"); StorageRead *read = NULL; @@ -680,6 +701,9 @@ testRun(void) "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" "x-amz-date: \n" "x-amz-security-token: \n" + "x-amz-server-side-encryption-customer-algorithm: AES256\n" + "x-amz-server-side-encryption-customer-key: \n" + "x-amz-server-side-encryption-customer-key-md5: \n" "*** Response Headers ***:\n" "content-length: 7\n" "*** Response Content ***:\n" @@ -713,7 +737,7 @@ testRun(void) testRequestP( service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz", - .kms = "kmskey1", .tag = "%20Key%202=%20Value%202&Key1=Value1"); + .kms = "kmskey1", .sseC = "rA1P", .tag = "%20Key%202=%20Value%202&Key1=Value1"); testResponseP(service); // Make a copy of the signing key to verify that it gets changed when the keys are updated @@ -746,7 +770,7 @@ testRun(void) TEST_TITLE("write zero-length file"); testRequestP( - service, s3, HTTP_VERB_PUT, "/file.txt", .content = "", .kms = "kmskey1", + service, s3, HTTP_VERB_PUT, "/file.txt", .content = "", .kms = "kmskey1", .sseC = "rA1P", .tag = "%20Key%202=%20Value%202&Key1=Value1"); testResponseP(service); @@ -757,7 +781,7 @@ testRun(void) TEST_TITLE("write file in chunks with nothing left over on close"); testRequestP( - service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", + service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P", .tag = "%20Key%202=%20Value%202&Key1=Value1"); testResponseP( service, @@ -769,10 +793,14 @@ testRun(void) "WxRt" ""); - testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456"); + testRequestP( + service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456", + .sseC = "rA1P"); testResponseP(service, .header = "etag:WxRt1"); - testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012"); + testRequestP( + service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012", + .sseC = "rA1P"); testResponseP(service, .header = "eTag:WxRt2"); testRequestP( @@ -798,7 +826,7 @@ testRun(void) // Stop writing tags driver->tag = NULL; - testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1"); + testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P"); testResponseP( service, .content = @@ -809,10 +837,14 @@ testRun(void) "WxRt" ""); - testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456"); + testRequestP( + service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456", + .sseC = "rA1P"); testResponseP(service, .header = "etag:WxRt1"); - testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012"); + testRequestP( + service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012", + .sseC = "rA1P"); testResponseP(service, .header = "eTag:WxRt2"); testRequestP( @@ -852,7 +884,7 @@ testRun(void) // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("write file in chunks with something left over on close"); - testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1"); + testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P"); testResponseP( service, .content = @@ -863,10 +895,14 @@ testRun(void) "RR55" ""); - testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456"); + testRequestP( + service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456", + .sseC = "rA1P"); testResponseP(service, .header = "etag:RR551"); - testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890"); + testRequestP( + service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890", + .sseC = "rA1P"); testResponseP(service, .header = "eTag:RR552"); testRequestP( @@ -889,7 +925,7 @@ testRun(void) // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("file missing"); - testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS"); + testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS", .sseC = "rA1P"); testResponseP(service, .code = 404); TEST_RESULT_BOOL(storageExistsP(s3, STRDEF("BOGUS")), false, "check");