1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-18 04:58:51 +02:00

Add support for AWS S3 server-side encryption using KMS.

AWS S3 integrates with AWS Key Management Service (AWS KMS) to provide server side encryption of S3 objects. This integration protects objects under encryption keys that never leave AWS KMS unencrypted.
This commit is contained in:
Christoph Berg 2022-01-13 14:46:14 +01:00 committed by GitHub
parent 92ea3e05fb
commit 3097acd73a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 179 additions and 17 deletions

View File

@ -16,6 +16,22 @@
<release-list>
<release date="XXXX-XX-XX" version="2.38dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-feature-list>
<release-item>
<github-issue id="1430"/>
<github-pull-request id="1567"/>
<release-item-contributor-list>
<release-item-contributor id="christoph.berg"/>
<release-item-reviewer id="david.steele"/>
<!-- Actually tester, but we don't have a tag for that yet -->
<release-item-reviewer id="tharindu.amila"/>
</release-item-contributor-list>
<p>Add support for <proper>AWS</proper> <proper>S3</proper> server-side encryption using <proper>KMS</proper>.</p>
</release-item>
</release-feature-list>
<release-improvement-list>
<release-item>
<github-pull-request id="1610"/>
@ -11196,6 +11212,11 @@
<contributor-id type="github">ktosiek</contributor-id>
</contributor>
<contributor id="tharindu.amila">
<contributor-name-display>Tharindu Amila</contributor-name-display>
<contributor-id type="github">tharinduamila-insta</contributor-id>
</contributor>
<contributor id="thomas.flatley">
<contributor-name-display>Thomas Flatley</contributor-name-display>
<contributor-id type="github">seadba</contributor-id>

View File

@ -2081,6 +2081,10 @@ option:
- auto
- web-id
repo-s3-kms-key-id:
inherit: repo-s3-bucket
required: false
repo-s3-region:
inherit: repo-s3-bucket
deprecate:

View File

@ -863,6 +863,17 @@
<example>AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22 ...</example>
</config-key>
<!-- CONFIG - REPO SECTION - REPO-S3-KMS-KEY-ID KEY -->
<config-key id="repo-s3-kms-key-id" name="S3 Repository KMS Key ID">
<summary>S3 repository KMS key.</summary>
<text>
<p>Setting this option enables S3 server-side encryption using the specified AWS key management service key.</p>
</text>
<example>bceb4f13-6939-4be3-910d-df54dee817b7</example>
</config-key>
<!-- CONFIG - REPO SECTION - REPO-S3-BUCKET KEY -->
<config-key id="repo-s3-bucket" name="S3 Repository Bucket">
<summary>S3 repository bucket.</summary>

View File

@ -126,7 +126,7 @@ Option constants
#define CFGOPT_TLS_SERVER_PORT "tls-server-port"
#define CFGOPT_TYPE "type"
#define CFG_OPTION_TOTAL 149
#define CFG_OPTION_TOTAL 150
/***********************************************************************************************************************************
Option value constants
@ -466,6 +466,7 @@ typedef enum
cfgOptRepoS3Key,
cfgOptRepoS3KeySecret,
cfgOptRepoS3KeyType,
cfgOptRepoS3KmsKeyId,
cfgOptRepoS3Region,
cfgOptRepoS3Role,
cfgOptRepoS3Token,

View File

@ -6795,6 +6795,83 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
),
),
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION
(
PARSE_RULE_OPTION_NAME("repo-s3-kms-key-id"),
PARSE_RULE_OPTION_TYPE(cfgOptTypeString),
PARSE_RULE_OPTION_RESET(true),
PARSE_RULE_OPTION_REQUIRED(false),
PARSE_RULE_OPTION_SECTION(cfgSectionGlobal),
PARSE_RULE_OPTION_GROUP_MEMBER(true),
PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo),
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup)
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck)
PARSE_RULE_OPTION_COMMAND(cfgCmdExpire)
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm)
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade)
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify)
),
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
),
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup)
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore)
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify)
),
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck)
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm)
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade)
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify)
),
PARSE_RULE_OPTIONAL
(
PARSE_RULE_OPTIONAL_GROUP
(
PARSE_RULE_OPTIONAL_DEPEND
(
PARSE_RULE_VAL_OPT(cfgOptRepoType),
PARSE_RULE_VAL_STRID(parseRuleValStrIdS3),
),
),
),
),
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION
(
@ -9108,6 +9185,7 @@ static const ConfigOption optionResolveOrder[] =
cfgOptRepoS3Bucket,
cfgOptRepoS3Endpoint,
cfgOptRepoS3KeyType,
cfgOptRepoS3KmsKeyId,
cfgOptRepoS3Region,
cfgOptRepoS3Role,
cfgOptRepoS3Token,

View File

@ -77,9 +77,10 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback,
cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx),
cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), role,
webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
FUNCTION_LOG_RETURN(STORAGE, result);
}

View File

@ -31,6 +31,9 @@ S3 HTTP headers
STRING_STATIC(S3_HEADER_CONTENT_SHA256_STR, "x-amz-content-sha256");
STRING_STATIC(S3_HEADER_DATE_STR, "x-amz-date");
STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-security-token");
STRING_STATIC(S3_HEADER_SRVSDENC_STR, "x-amz-server-side-encryption");
STRING_STATIC(S3_HEADER_SRVSDENC_KMS_STR, "aws:kms");
STRING_STATIC(S3_HEADER_SRVSDENC_KMSKEYID_STR, "x-amz-server-side-encryption-aws-kms-key-id");
/***********************************************************************************************************************************
S3 query tokens
@ -90,6 +93,7 @@ struct StorageS3
String *accessKey; // Access key
String *secretAccessKey; // Secret access key
String *securityToken; // Security token, if any
const String *kmsKeyId; // Server-side encryption key
size_t partSize; // Part size for multi-part upload
unsigned int deleteMax; // Maximum objects that can be deleted in one request
StorageS3UriStyle uriStyle; // Path or host style URIs
@ -423,6 +427,7 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_PARAM(BOOL, param.sseKms);
FUNCTION_LOG_END();
ASSERT(this != NULL);
@ -449,6 +454,13 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S
strNewEncode(encodeBase64, cryptoHashOne(HASH_TYPE_MD5_STR, param.content)));
}
// Set KMS headers when requested
if (param.sseKms && this->kmsKeyId != NULL)
{
httpHeaderPut(requestHeader, S3_HEADER_SRVSDENC_STR, S3_HEADER_SRVSDENC_KMS_STR);
httpHeaderPut(requestHeader, S3_HEADER_SRVSDENC_KMSKEYID_STR, this->kmsKeyId);
}
// When using path-style URIs the bucket name needs to be prepended
if (this->uriStyle == storageS3UriStylePath)
path = strNewFmt("/%s%s", strZ(this->bucket), strZ(path));
@ -552,12 +564,14 @@ storageS3Request(StorageS3 *this, const String *verb, const String *path, Storag
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_PARAM(BOOL, param.allowMissing);
FUNCTION_LOG_PARAM(BOOL, param.contentIo);
FUNCTION_LOG_PARAM(BOOL, param.sseKms);
FUNCTION_LOG_END();
FUNCTION_LOG_RETURN(
HTTP_RESPONSE,
storageS3ResponseP(
storageS3RequestAsyncP(this, verb, path, .header = param.header, .query = param.query, .content = param.content),
storageS3RequestAsyncP(
this, verb, path, .header = param.header, .query = param.query, .content = param.content, .sseKms = param.sseKms),
.allowMissing = param.allowMissing, .contentIo = param.contentIo));
}
@ -1021,9 +1035,9 @@ Storage *
storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
const String *secretAccessKey, const String *securityToken, const String *credRole, const String *const webIdToken,
size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile,
const String *caPath)
const String *secretAccessKey, const String *securityToken, const String *const kmsKeyId, const String *credRole,
const String *const webIdToken, size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer,
const String *caFile, const String *caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@ -1037,6 +1051,7 @@ storageS3New(
FUNCTION_TEST_PARAM(STRING, accessKey);
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
FUNCTION_TEST_PARAM(STRING, securityToken);
FUNCTION_TEST_PARAM(STRING, kmsKeyId);
FUNCTION_TEST_PARAM(STRING, credRole);
FUNCTION_TEST_PARAM(STRING, webIdToken);
FUNCTION_LOG_PARAM(SIZE, partSize);
@ -1066,6 +1081,7 @@ storageS3New(
.bucket = strDup(bucket),
.region = strDup(region),
.keyType = keyType,
.kmsKeyId = strDup(kmsKeyId),
.partSize = partSize,
.deleteMax = STORAGE_S3_DELETE_MAX,
.uriStyle = uriStyle,

View File

@ -41,7 +41,8 @@ Constructors
Storage *storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
const String *secretAccessKey, const String *securityToken, const String *credRole, const String *webIdToken, size_t partSize,
const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
const String *secretAccessKey, const String *securityToken, const String *kmsKeyId, const String *credRole,
const String *webIdToken, size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer,
const String *caFile, const String *caPath);
#endif

View File

@ -22,6 +22,7 @@ typedef struct StorageS3RequestAsyncParam
const HttpHeader *header; // Headers
const HttpQuery *query; // Query parameters
const Buffer *content; // Request content
bool sseKms; // Enable server-side encryption?
} StorageS3RequestAsyncParam;
#define storageS3RequestAsyncP(this, verb, path, ...) \
@ -51,6 +52,7 @@ typedef struct StorageS3RequestParam
const Buffer *content; // Request content
bool allowMissing; // Allow missing files (caller can check response code)
bool contentIo; // Is IoRead interface required to read content?
bool sseKms; // Enable server-side encryption?
} StorageS3RequestParam;
#define storageS3RequestP(this, verb, path, ...) \

View File

@ -125,7 +125,7 @@ storageWriteS3PartAsync(StorageWriteS3 *this)
httpResponseContent(
storageS3RequestP(
this->storage, HTTP_VERB_POST_STR, this->interface.name,
.query = httpQueryAdd(httpQueryNewP(), S3_QUERY_UPLOADS_STR, EMPTY_STR)))));
.query = httpQueryAdd(httpQueryNewP(), S3_QUERY_UPLOADS_STR, EMPTY_STR), .sseKms = true))));
// Store the upload id
MEM_CONTEXT_BEGIN(THIS_MEM_CONTEXT())
@ -248,7 +248,10 @@ storageWriteS3Close(THIS_VOID)
}
// Else upload all the data in a single put
else
storageS3RequestP(this->storage, HTTP_VERB_PUT_STR, this->interface.name, .content = this->partBuffer);
{
storageS3RequestP(
this->storage, HTTP_VERB_PUT_STR, this->interface.name, .content = this->partBuffer, .sseKms = true);
}
bufFree(this->partBuffer);
this->partBuffer = NULL;

View File

@ -297,6 +297,7 @@ testRun(void)
" --repo-s3-key S3 repository access key\n"
" --repo-s3-key-secret S3 repository secret access key\n"
" --repo-s3-key-type S3 repository key type [default=shared]\n"
" --repo-s3-kms-key-id S3 repository KMS key\n"
" --repo-s3-region S3 repository region\n"
" --repo-s3-role S3 repository role\n"
" --repo-s3-token S3 repository security token\n"

View File

@ -28,6 +28,7 @@ typedef struct TestRequestParam
const char *accessKey;
const char *securityToken;
const char *range;
const char *kms;
} TestRequestParam;
#define testRequestP(write, s3, verb, path, ...) \
@ -75,6 +76,9 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
if (securityToken != NULL)
strCatZ(request, ";x-amz-security-token");
if (param.kms != NULL)
strCatZ(request, ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id");
strCatZ(request, ",Signature=????????????????????????????????????????????????????????????????\r\n");
}
@ -120,6 +124,13 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
strCatFmt(request, "x-amz-security-token:%s\r\n", securityToken);
}
// Add kms key
if (param.kms != NULL)
{
strCatZ(request, "x-amz-server-side-encryption:aws:kms\r\n");
strCatFmt(request, "x-amz-server-side-encryption-aws-kms-key-id:%s\r\n", param.kms);
}
// Add final \r\n
strCatZ(request, "\r\n");
@ -453,6 +464,7 @@ testRun(void)
hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), port);
hrnCfgArgRaw(argList, cfgOptRepoS3Role, credRole);
hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeAuto);
hrnCfgArgRawZ(argList, cfgOptRepoS3KmsKeyId, "kmskey1");
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
s3 = storageRepoGet(0, true);
@ -628,7 +640,9 @@ testRun(void)
hrnServerScriptClose(auth);
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz",
.kms = "kmskey1");
testResponseP(service);
// Make a copy of the signing key to verify that it gets changed when the keys are updated
@ -659,7 +673,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write zero-length file");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "", .kms = "kmskey1");
testResponseP(service);
TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
@ -668,7 +682,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with nothing left over on close");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1");
testResponseP(
service,
.content =
@ -705,7 +719,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error in success response of multipart upload");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1");
testResponseP(
service,
.content =
@ -759,7 +773,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with something left over on close");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1");
testResponseP(
service,
.content =
@ -909,6 +923,15 @@ testRun(void)
// Auth service no longer needed
hrnServerScriptEnd(auth);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write zero-length file (without kms)");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "");
testResponseP(service);
TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
TEST_RESULT_VOID(storagePutP(write, NULL), "write");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("info check existence only");