diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 24f8f393b..976bcc72c 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -16,6 +16,22 @@ + + + + + + + + + + + + +

Add support for AWS S3 server-side encryption using KMS.

+ + + @@ -11196,6 +11212,11 @@ ktosiek + + Tharindu Amila + tharinduamila-insta + + Thomas Flatley seadba diff --git a/src/build/config/config.yaml b/src/build/config/config.yaml index 19d4907c8..995b50cc7 100644 --- a/src/build/config/config.yaml +++ b/src/build/config/config.yaml @@ -2081,6 +2081,10 @@ option: - auto - web-id + repo-s3-kms-key-id: + inherit: repo-s3-bucket + required: false + repo-s3-region: inherit: repo-s3-bucket deprecate: diff --git a/src/build/help/help.xml b/src/build/help/help.xml index 2154048ee..ce1bf5194 100644 --- a/src/build/help/help.xml +++ b/src/build/help/help.xml @@ -863,6 +863,17 @@ AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22 ... + + + S3 repository KMS key. + + +

Setting this option enables S3 server-side encryption using the specified AWS key management service key.

+ + + bceb4f13-6939-4be3-910d-df54dee817b7 + + S3 repository bucket. diff --git a/src/config/config.auto.h b/src/config/config.auto.h index ee343aa0a..ff33b54c5 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -126,7 +126,7 @@ Option constants #define CFGOPT_TLS_SERVER_PORT "tls-server-port" #define CFGOPT_TYPE "type" -#define CFG_OPTION_TOTAL 149 +#define CFG_OPTION_TOTAL 150 /*********************************************************************************************************************************** Option value constants @@ -466,6 +466,7 @@ typedef enum cfgOptRepoS3Key, cfgOptRepoS3KeySecret, cfgOptRepoS3KeyType, + cfgOptRepoS3KmsKeyId, cfgOptRepoS3Region, cfgOptRepoS3Role, cfgOptRepoS3Token, diff --git a/src/config/parse.auto.c b/src/config/parse.auto.c index a360b1647..a70c175a2 100644 --- a/src/config/parse.auto.c +++ b/src/config/parse.auto.c @@ -6795,6 +6795,83 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = ), ), + // ----------------------------------------------------------------------------------------------------------------------------- + PARSE_RULE_OPTION + ( + PARSE_RULE_OPTION_NAME("repo-s3-kms-key-id"), + PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_RESET(true), + PARSE_RULE_OPTION_REQUIRED(false), + PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), + PARSE_RULE_OPTION_GROUP_MEMBER(true), + PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo), + + PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST + ( + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) + PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) + PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) + PARSE_RULE_OPTION_COMMAND(cfgCmdExpire) + PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) + ), + + PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST + ( + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) + ), + + PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST + ( + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) + PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) + ), + + PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST + ( + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) + PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) + PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) + ), + + PARSE_RULE_OPTIONAL + ( + PARSE_RULE_OPTIONAL_GROUP + ( + PARSE_RULE_OPTIONAL_DEPEND + ( + PARSE_RULE_VAL_OPT(cfgOptRepoType), + PARSE_RULE_VAL_STRID(parseRuleValStrIdS3), + ), + ), + ), + ), + // ----------------------------------------------------------------------------------------------------------------------------- PARSE_RULE_OPTION ( @@ -9108,6 +9185,7 @@ static const ConfigOption optionResolveOrder[] = cfgOptRepoS3Bucket, cfgOptRepoS3Endpoint, cfgOptRepoS3KeyType, + cfgOptRepoS3KmsKeyId, cfgOptRepoS3Region, cfgOptRepoS3Role, cfgOptRepoS3Token, diff --git a/src/storage/s3/helper.c b/src/storage/s3/helper.c index bd73d85f7..d0d14bb5a 100644 --- a/src/storage/s3/helper.c +++ b/src/storage/s3/helper.c @@ -77,9 +77,10 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), role, - webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); + cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), + cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), + cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), + cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); FUNCTION_LOG_RETURN(STORAGE, result); } diff --git a/src/storage/s3/storage.c b/src/storage/s3/storage.c index 8dd46b492..4e8766e22 100644 --- a/src/storage/s3/storage.c +++ b/src/storage/s3/storage.c @@ -31,6 +31,9 @@ S3 HTTP headers STRING_STATIC(S3_HEADER_CONTENT_SHA256_STR, "x-amz-content-sha256"); STRING_STATIC(S3_HEADER_DATE_STR, "x-amz-date"); STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-security-token"); +STRING_STATIC(S3_HEADER_SRVSDENC_STR, "x-amz-server-side-encryption"); +STRING_STATIC(S3_HEADER_SRVSDENC_KMS_STR, "aws:kms"); +STRING_STATIC(S3_HEADER_SRVSDENC_KMSKEYID_STR, "x-amz-server-side-encryption-aws-kms-key-id"); /*********************************************************************************************************************************** S3 query tokens @@ -90,6 +93,7 @@ struct StorageS3 String *accessKey; // Access key String *secretAccessKey; // Secret access key String *securityToken; // Security token, if any + const String *kmsKeyId; // Server-side encryption key size_t partSize; // Part size for multi-part upload unsigned int deleteMax; // Maximum objects that can be deleted in one request StorageS3UriStyle uriStyle; // Path or host style URIs @@ -423,6 +427,7 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S FUNCTION_LOG_PARAM(HTTP_HEADER, param.header); FUNCTION_LOG_PARAM(HTTP_QUERY, param.query); FUNCTION_LOG_PARAM(BUFFER, param.content); + FUNCTION_LOG_PARAM(BOOL, param.sseKms); FUNCTION_LOG_END(); ASSERT(this != NULL); @@ -449,6 +454,13 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S strNewEncode(encodeBase64, cryptoHashOne(HASH_TYPE_MD5_STR, param.content))); } + // Set KMS headers when requested + if (param.sseKms && this->kmsKeyId != NULL) + { + httpHeaderPut(requestHeader, S3_HEADER_SRVSDENC_STR, S3_HEADER_SRVSDENC_KMS_STR); + httpHeaderPut(requestHeader, S3_HEADER_SRVSDENC_KMSKEYID_STR, this->kmsKeyId); + } + // When using path-style URIs the bucket name needs to be prepended if (this->uriStyle == storageS3UriStylePath) path = strNewFmt("/%s%s", strZ(this->bucket), strZ(path)); @@ -552,12 +564,14 @@ storageS3Request(StorageS3 *this, const String *verb, const String *path, Storag FUNCTION_LOG_PARAM(BUFFER, param.content); FUNCTION_LOG_PARAM(BOOL, param.allowMissing); FUNCTION_LOG_PARAM(BOOL, param.contentIo); + FUNCTION_LOG_PARAM(BOOL, param.sseKms); FUNCTION_LOG_END(); FUNCTION_LOG_RETURN( HTTP_RESPONSE, storageS3ResponseP( - storageS3RequestAsyncP(this, verb, path, .header = param.header, .query = param.query, .content = param.content), + storageS3RequestAsyncP( + this, verb, path, .header = param.header, .query = param.query, .content = param.content, .sseKms = param.sseKms), .allowMissing = param.allowMissing, .contentIo = param.contentIo)); } @@ -1021,9 +1035,9 @@ Storage * storageS3New( const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey, - const String *secretAccessKey, const String *securityToken, const String *credRole, const String *const webIdToken, - size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, - const String *caPath) + const String *secretAccessKey, const String *securityToken, const String *const kmsKeyId, const String *credRole, + const String *const webIdToken, size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, + const String *caFile, const String *caPath) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, path); @@ -1037,6 +1051,7 @@ storageS3New( FUNCTION_TEST_PARAM(STRING, accessKey); FUNCTION_TEST_PARAM(STRING, secretAccessKey); FUNCTION_TEST_PARAM(STRING, securityToken); + FUNCTION_TEST_PARAM(STRING, kmsKeyId); FUNCTION_TEST_PARAM(STRING, credRole); FUNCTION_TEST_PARAM(STRING, webIdToken); FUNCTION_LOG_PARAM(SIZE, partSize); @@ -1066,6 +1081,7 @@ storageS3New( .bucket = strDup(bucket), .region = strDup(region), .keyType = keyType, + .kmsKeyId = strDup(kmsKeyId), .partSize = partSize, .deleteMax = STORAGE_S3_DELETE_MAX, .uriStyle = uriStyle, diff --git a/src/storage/s3/storage.h b/src/storage/s3/storage.h index 6d6b42b9b..4e19933c0 100644 --- a/src/storage/s3/storage.h +++ b/src/storage/s3/storage.h @@ -41,7 +41,8 @@ Constructors Storage *storageS3New( const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey, - const String *secretAccessKey, const String *securityToken, const String *credRole, const String *webIdToken, size_t partSize, - const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath); + const String *secretAccessKey, const String *securityToken, const String *kmsKeyId, const String *credRole, + const String *webIdToken, size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, + const String *caFile, const String *caPath); #endif diff --git a/src/storage/s3/storage.intern.h b/src/storage/s3/storage.intern.h index 69fe98c44..59d5e10e5 100644 --- a/src/storage/s3/storage.intern.h +++ b/src/storage/s3/storage.intern.h @@ -22,6 +22,7 @@ typedef struct StorageS3RequestAsyncParam const HttpHeader *header; // Headers const HttpQuery *query; // Query parameters const Buffer *content; // Request content + bool sseKms; // Enable server-side encryption? } StorageS3RequestAsyncParam; #define storageS3RequestAsyncP(this, verb, path, ...) \ @@ -51,6 +52,7 @@ typedef struct StorageS3RequestParam const Buffer *content; // Request content bool allowMissing; // Allow missing files (caller can check response code) bool contentIo; // Is IoRead interface required to read content? + bool sseKms; // Enable server-side encryption? } StorageS3RequestParam; #define storageS3RequestP(this, verb, path, ...) \ diff --git a/src/storage/s3/write.c b/src/storage/s3/write.c index 08b19cf34..64b6b5c98 100644 --- a/src/storage/s3/write.c +++ b/src/storage/s3/write.c @@ -125,7 +125,7 @@ storageWriteS3PartAsync(StorageWriteS3 *this) httpResponseContent( storageS3RequestP( this->storage, HTTP_VERB_POST_STR, this->interface.name, - .query = httpQueryAdd(httpQueryNewP(), S3_QUERY_UPLOADS_STR, EMPTY_STR))))); + .query = httpQueryAdd(httpQueryNewP(), S3_QUERY_UPLOADS_STR, EMPTY_STR), .sseKms = true)))); // Store the upload id MEM_CONTEXT_BEGIN(THIS_MEM_CONTEXT()) @@ -248,7 +248,10 @@ storageWriteS3Close(THIS_VOID) } // Else upload all the data in a single put else - storageS3RequestP(this->storage, HTTP_VERB_PUT_STR, this->interface.name, .content = this->partBuffer); + { + storageS3RequestP( + this->storage, HTTP_VERB_PUT_STR, this->interface.name, .content = this->partBuffer, .sseKms = true); + } bufFree(this->partBuffer); this->partBuffer = NULL; diff --git a/test/src/module/command/helpTest.c b/test/src/module/command/helpTest.c index 2ed4294c9..094a3f4fe 100644 --- a/test/src/module/command/helpTest.c +++ b/test/src/module/command/helpTest.c @@ -297,6 +297,7 @@ testRun(void) " --repo-s3-key S3 repository access key\n" " --repo-s3-key-secret S3 repository secret access key\n" " --repo-s3-key-type S3 repository key type [default=shared]\n" + " --repo-s3-kms-key-id S3 repository KMS key\n" " --repo-s3-region S3 repository region\n" " --repo-s3-role S3 repository role\n" " --repo-s3-token S3 repository security token\n" diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index 54dd3c69e..962c0e453 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -28,6 +28,7 @@ typedef struct TestRequestParam const char *accessKey; const char *securityToken; const char *range; + const char *kms; } TestRequestParam; #define testRequestP(write, s3, verb, path, ...) \ @@ -75,6 +76,9 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes if (securityToken != NULL) strCatZ(request, ";x-amz-security-token"); + if (param.kms != NULL) + strCatZ(request, ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id"); + strCatZ(request, ",Signature=????????????????????????????????????????????????????????????????\r\n"); } @@ -120,6 +124,13 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes strCatFmt(request, "x-amz-security-token:%s\r\n", securityToken); } + // Add kms key + if (param.kms != NULL) + { + strCatZ(request, "x-amz-server-side-encryption:aws:kms\r\n"); + strCatFmt(request, "x-amz-server-side-encryption-aws-kms-key-id:%s\r\n", param.kms); + } + // Add final \r\n strCatZ(request, "\r\n"); @@ -453,6 +464,7 @@ testRun(void) hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), port); hrnCfgArgRaw(argList, cfgOptRepoS3Role, credRole); hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeAuto); + hrnCfgArgRawZ(argList, cfgOptRepoS3KmsKeyId, "kmskey1"); HRN_CFG_LOAD(cfgCmdArchivePush, argList); s3 = storageRepoGet(0, true); @@ -628,7 +640,9 @@ testRun(void) hrnServerScriptClose(auth); - testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz"); + testRequestP( + service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz", + .kms = "kmskey1"); testResponseP(service); // Make a copy of the signing key to verify that it gets changed when the keys are updated @@ -659,7 +673,7 @@ testRun(void) // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("write zero-length file"); - testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = ""); + testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "", .kms = "kmskey1"); testResponseP(service); TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write"); @@ -668,7 +682,7 @@ testRun(void) // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("write file in chunks with nothing left over on close"); - testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads="); + testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1"); testResponseP( service, .content = @@ -705,7 +719,7 @@ testRun(void) // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("error in success response of multipart upload"); - testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads="); + testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1"); testResponseP( service, .content = @@ -759,7 +773,7 @@ testRun(void) // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("write file in chunks with something left over on close"); - testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads="); + testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1"); testResponseP( service, .content = @@ -909,6 +923,15 @@ testRun(void) // Auth service no longer needed hrnServerScriptEnd(auth); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("write zero-length file (without kms)"); + + testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = ""); + testResponseP(service); + + TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write"); + TEST_RESULT_VOID(storagePutP(write, NULL), "write"); + // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("info check existence only");