db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

S3 SSE-C encryption support.

Browse Source 
This feature (enabled with --repo-s3-sse-customer-key) provides an encryption key to encrypt the data after it has been transmitted to the server.

While not as secure as encrypting data before transmission (--repo-cipher-type), this may be useful in certain configurations.
This commit is contained in:
udf2457
2024-04-17 02:58:13 +01:00
committed by GitHub
parent 06d3cb767c
commit 57731b6cd2
15 changed files with 215 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/xml/release/2024/2.52.xml
+15
View File
@@ -1,5 +1,20 @@
<release date="XXXX-XX-XX" version="2.52dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-feature-list>
<release-item>
<github-issue id="2279"/>
<github-pull-request id="2282"/>
<release-item-contributor-list>
<release-item-ideator id="udf2457"/>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="udf2457"/>
</release-item-contributor-list>
<p><proper>S3</proper> <id>SSE-C</id> encryption support.</p>
</release-item>
</release-feature-list>
<release-improvement-list>
<release-item>
<github-pull-request id="2321"/>
doc/xml/release/contributor.xml
+5
View File
@@ -1020,6 +1020,11 @@
<contributor-id type="github">ucando</contributor-id>
</contributor>
<contributor id="udf2457">
<contributor-name-display>udf2457</contributor-name-display>
<contributor-id type="github">udf2457</contributor-id>
</contributor>
<contributor id="ugo.bellavance">
<contributor-name-display>Ugo Bellavance</contributor-name-display>
</contributor>
src/build/config/config.yaml
+5
View File
@@ -2333,6 +2333,11 @@ option:
inherit: repo-s3-bucket
required: false
repo-s3-sse-customer-key:
inherit: repo-s3-bucket
required: false
secure: true
repo-s3-region:
inherit: repo-s3-bucket
deprecate:
src/build/help/help.xml
+10
View File
@@ -1044,6 +1044,16 @@
<example>us-east-1</example>
</config-key>
<config-key id="repo-s3-sse-customer-key" name="S3 Repository SSE Customer Key">
<summary>S3 Repository SSE Customer Key.</summary>
<text>
<p>Setting this option enables S3 server-side encryption using the specified customer key.</p>
</text>
<example>bceb4f13-6939-4be3-910d-df54dee817b7</example>
</config-key>
<config-key id="repo-s3-uri-style" name="S3 Repository URI Style">
<summary>S3 URI Style.</summary>
src/config/config.auto.h
+2 -1
View File
@@ -136,7 +136,7 @@ Option constants
#define CFGOPT_TYPE "type"
#define CFGOPT_VERBOSE "verbose"
#define CFG_OPTION_TOTAL 179
#define CFG_OPTION_TOTAL 180
/***********************************************************************************************************************************
Option value constants
@@ -516,6 +516,7 @@ typedef enum
cfgOptRepoS3KmsKeyId,
cfgOptRepoS3Region,
cfgOptRepoS3Role,
cfgOptRepoS3SseCustomerKey,
cfgOptRepoS3Token,
cfgOptRepoS3UriStyle,
cfgOptRepoSftpHost,
src/config/parse.auto.c.inc
+82
View File
@@ -7723,6 +7723,87 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
), // opt/repo-s3-role
), // opt/repo-s3-role
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-s3-sse-customer-key
( // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_NAME("repo-s3-sse-customer-key"), // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_TYPE(cfgOptTypeString), // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_RESET(true), // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_REQUIRED(false), // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_SECURE(true), // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_GROUP_MEMBER(true), // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo), // opt/repo-s3-sse-customer-key
// opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-s3-sse-customer-key
( // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdAnnotate) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdExpire) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdManifest) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-s3-sse-customer-key
), // opt/repo-s3-sse-customer-key
// opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/repo-s3-sse-customer-key
( // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-s3-sse-customer-key
), // opt/repo-s3-sse-customer-key
// opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/repo-s3-sse-customer-key
( // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-s3-sse-customer-key
), // opt/repo-s3-sse-customer-key
// opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/repo-s3-sse-customer-key
( // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdAnnotate) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdManifest) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-s3-sse-customer-key
), // opt/repo-s3-sse-customer-key
// opt/repo-s3-sse-customer-key
PARSE_RULE_OPTIONAL // opt/repo-s3-sse-customer-key
( // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTIONAL_GROUP // opt/repo-s3-sse-customer-key
( // opt/repo-s3-sse-customer-key
PARSE_RULE_OPTIONAL_DEPEND // opt/repo-s3-sse-customer-key
( // opt/repo-s3-sse-customer-key
PARSE_RULE_VAL_OPT(cfgOptRepoType), // opt/repo-s3-sse-customer-key
PARSE_RULE_VAL_STRID(parseRuleValStrIdS3), // opt/repo-s3-sse-customer-key
), // opt/repo-s3-sse-customer-key
), // opt/repo-s3-sse-customer-key
), // opt/repo-s3-sse-customer-key
), // opt/repo-s3-sse-customer-key
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-s3-token
( // opt/repo-s3-token
PARSE_RULE_OPTION_NAME("repo-s3-token"), // opt/repo-s3-token
@@ -10977,6 +11058,7 @@ static const uint8_t optionResolveOrder[] =
cfgOptRepoS3KmsKeyId, // opt-resolve-order
cfgOptRepoS3Region, // opt-resolve-order
cfgOptRepoS3Role, // opt-resolve-order
cfgOptRepoS3SseCustomerKey, // opt-resolve-order
cfgOptRepoS3Token, // opt-resolve-order
cfgOptRepoS3UriStyle, // opt-resolve-order
cfgOptRepoSftpHost, // opt-resolve-order
src/storage/s3/helper.c
+3 -2
View File
@@ -85,8 +85,9 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint,
(StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx),
keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role,
webIdToken, (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3SseCustomerKey, repoIdx), role, webIdToken,
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx),
cfgOptionIdxKvNull(cfgOptRepoStorageTag, repoIdx), host, port, ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
src/storage/s3/read.c
+1 -1
View File
@@ -55,7 +55,7 @@ storageReadS3Open(THIS_VOID)
this->httpResponse = storageS3RequestP(
this->storage, HTTP_VERB_GET_STR, this->interface.name,
.header = httpHeaderPutRange(httpHeaderNew(NULL), this->interface.offset, this->interface.limit),
.allowMissing = true, .contentIo = true);
.allowMissing = true, .contentIo = true, .sseC = true);
}
MEM_CONTEXT_OBJ_END();
src/storage/s3/storage.c
+32 -5
View File
@@ -33,6 +33,10 @@ STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-secur
STRING_STATIC(S3_HEADER_SRVSDENC_STR, "x-amz-server-side-encryption");
STRING_STATIC(S3_HEADER_SRVSDENC_KMS_STR, "aws:kms");
STRING_STATIC(S3_HEADER_SRVSDENC_KMSKEYID_STR, "x-amz-server-side-encryption-aws-kms-key-id");
STRING_STATIC(S3_HEADER_SSECUSTKEY_ALGO_STR, "x-amz-server-side-encryption-customer-algorithm");
STRING_STATIC(S3_HEADER_SSECUSTKEY_AES256_STR, "AES256");
STRING_STATIC(S3_HEADER_SSECUSTKEY_KEY_STR, "x-amz-server-side-encryption-customer-key");
STRING_STATIC(S3_HEADER_SSECUSTKEY_KEY_MD5_STR, "x-amz-server-side-encryption-customer-key-md5");
STRING_STATIC(S3_HEADER_TAGGING, "x-amz-tagging");
/***********************************************************************************************************************************
@@ -94,6 +98,8 @@ struct StorageS3
String *secretAccessKey; // Secret access key
String *securityToken; // Security token, if any
const String *kmsKeyId; // Server-side encryption key
const String *sseCustomerKey; // Base64 of SSE-C encryption key
const String *sseCustomerKeyMd5; // Base64 of MD5 of SSE-C key
size_t partSize; // Part size for multi-part upload
const String *tag; // Tags to be applied to objects
unsigned int deleteMax; // Maximum objects that can be deleted in one request
@@ -455,6 +461,7 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_PARAM(BOOL, param.sseKms);
FUNCTION_LOG_PARAM(BOOL, param.sseC);
FUNCTION_LOG_PARAM(BOOL, param.tag);
FUNCTION_LOG_END();
@@ -489,6 +496,14 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S
httpHeaderPut(requestHeader, S3_HEADER_SRVSDENC_KMSKEYID_STR, this->kmsKeyId);
}
// Set SSE-C headers when requested
if (param.sseC && this->sseCustomerKey != NULL)
{
httpHeaderPut(requestHeader, S3_HEADER_SSECUSTKEY_ALGO_STR, S3_HEADER_SSECUSTKEY_AES256_STR);
httpHeaderPut(requestHeader, S3_HEADER_SSECUSTKEY_KEY_STR, this->sseCustomerKey);
httpHeaderPut(requestHeader, S3_HEADER_SSECUSTKEY_KEY_MD5_STR, this->sseCustomerKeyMd5);
}
// Set tags when requested and available
if (param.tag && this->tag != NULL)
httpHeaderPut(requestHeader, S3_HEADER_TAGGING, this->tag);
@@ -599,12 +614,13 @@ storageS3Request(StorageS3 *this, const String *verb, const String *path, Storag
FUNCTION_LOG_PARAM(BOOL, param.allowMissing);
FUNCTION_LOG_PARAM(BOOL, param.contentIo);
FUNCTION_LOG_PARAM(BOOL, param.sseKms);
FUNCTION_LOG_PARAM(BOOL, param.sseC);
FUNCTION_LOG_PARAM(BOOL, param.tag);
FUNCTION_LOG_END();
HttpRequest *const request = storageS3RequestAsyncP(
this, verb, path, .header = param.header, .query = param.query, .content = param.content, .sseKms = param.sseKms,
.tag = param.tag);
.sseC = param.sseC, .tag = param.tag);
HttpResponse *const result = storageS3ResponseP(
request, .allowMissing = param.allowMissing, .contentIo = param.contentIo);
@@ -801,7 +817,7 @@ storageS3Info(THIS_VOID, const String *const file, const StorageInfoLevel level,
ASSERT(file != NULL);
// Attempt to get file info
HttpResponse *const httpResponse = storageS3RequestP(this, HTTP_VERB_HEAD_STR, file, .allowMissing = true);
HttpResponse *const httpResponse = storageS3RequestP(this, HTTP_VERB_HEAD_STR, file, .allowMissing = true, .sseC = true);
// Does the file exist?
StorageInfo result = {.level = level, .exists = httpResponseCodeOk(httpResponse)};
@@ -1105,9 +1121,9 @@ storageS3New(
const String *const path, const bool write, StoragePathExpressionCallback pathExpressionFunction, const String *const bucket,
const String *const endPoint, const StorageS3UriStyle uriStyle, const String *const region, const StorageS3KeyType keyType,
const String *const accessKey, const String *const secretAccessKey, const String *const securityToken,
const String *const kmsKeyId, const String *const credRole, const String *const webIdToken, const size_t partSize,
const KeyValue *const tag, const String *host, const unsigned int port, const TimeMSec timeout, const bool verifyPeer,
const String *const caFile, const String *const caPath)
const String *const kmsKeyId, const String *sseCustomerKey, const String *const credRole, const String *const webIdToken,
const size_t partSize, const KeyValue *const tag, const String *host, const unsigned int port, const TimeMSec timeout,
const bool verifyPeer, const String *const caFile, const String *const caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@@ -1122,6 +1138,7 @@ storageS3New(
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
FUNCTION_TEST_PARAM(STRING, securityToken);
FUNCTION_TEST_PARAM(STRING, kmsKeyId);
FUNCTION_TEST_PARAM(STRING, sseCustomerKey);
FUNCTION_TEST_PARAM(STRING, credRole);
FUNCTION_TEST_PARAM(STRING, webIdToken);
FUNCTION_LOG_PARAM(SIZE, partSize);
@@ -1149,6 +1166,7 @@ storageS3New(
.region = strDup(region),
.keyType = keyType,
.kmsKeyId = strDup(kmsKeyId),
.sseCustomerKey = strDup(sseCustomerKey),
.partSize = partSize,
.deleteMax = STORAGE_S3_DELETE_MAX,
.uriStyle = uriStyle,
@@ -1227,10 +1245,19 @@ storageS3New(
}
}
// Generate SSE customer key MD5 hash
if (this->sseCustomerKey != NULL)
{
this->sseCustomerKeyMd5 = strNewEncode(
encodingBase64, cryptoHashOne(hashTypeMd5, bufNewDecode(encodingBase64, this->sseCustomerKey)));
}
// Create list of redacted headers
this->headerRedactList = strLstNew();
strLstAdd(this->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
strLstAdd(this->headerRedactList, S3_HEADER_DATE_STR);
strLstAdd(this->headerRedactList, S3_HEADER_SSECUSTKEY_KEY_STR);
strLstAdd(this->headerRedactList, S3_HEADER_SSECUSTKEY_KEY_MD5_STR);
strLstAdd(this->headerRedactList, S3_HEADER_TOKEN_STR);
}
OBJ_NEW_END();
src/storage/s3/storage.h
+3 -3
View File
@@ -36,8 +36,8 @@ Constructors
FN_EXTERN Storage *storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
const String *secretAccessKey, const String *securityToken, const String *kmsKeyId, const String *credRole,
const String *webIdToken, size_t partSize, const KeyValue *tag, const String *host, unsigned int port, TimeMSec timeout,
bool verifyPeer, const String *caFile, const String *caPath);
const String *secretAccessKey, const String *securityToken, const String *kmsKeyId, const String *sseCustomerKey,
const String *credRole, const String *webIdToken, size_t partSize, const KeyValue *tag, const String *host, unsigned int port,
TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
#endif
src/storage/s3/storage.intern.h
+2
View File
@@ -23,6 +23,7 @@ typedef struct StorageS3RequestAsyncParam
const HttpQuery *query; // Query parameters
const Buffer *content; // Request content
bool sseKms; // Enable server-side encryption?
bool sseC; // Enable server-side encryption with customer-provided keys?
bool tag; // Add tags when available?
} StorageS3RequestAsyncParam;
@@ -54,6 +55,7 @@ typedef struct StorageS3RequestParam
bool allowMissing; // Allow missing files (caller can check response code)
bool contentIo; // Is IoRead interface required to read content?
bool sseKms; // Enable server-side encryption?
bool sseC; // Enable server-side encryption with customer-provided keys?
bool tag; // Add tags when available?
} StorageS3RequestParam;
src/storage/s3/write.c
+3 -3
View File
@@ -128,7 +128,7 @@ storageWriteS3PartAsync(StorageWriteS3 *this)
storageS3RequestP(
this->storage, HTTP_VERB_POST_STR, this->interface.name,
.query = httpQueryAdd(httpQueryNewP(), S3_QUERY_UPLOADS_STR, EMPTY_STR), .sseKms = true,
.tag = true))));
.sseC = true, .tag = true))));
// Store the upload id
MEM_CONTEXT_OBJ_BEGIN(this)
@@ -147,7 +147,7 @@ storageWriteS3PartAsync(StorageWriteS3 *this)
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->request = storageS3RequestAsyncP(
this->storage, HTTP_VERB_PUT_STR, this->interface.name, .query = query, .content = this->partBuffer);
this->storage, HTTP_VERB_PUT_STR, this->interface.name, .query = query, .content = this->partBuffer, .sseC = true);
}
MEM_CONTEXT_OBJ_END();
}
@@ -256,7 +256,7 @@ storageWriteS3Close(THIS_VOID)
{
storageS3RequestP(
this->storage, HTTP_VERB_PUT_STR, this->interface.name, .content = this->partBuffer, .sseKms = true,
.tag = true);
.sseC = true, .tag = true);
}
bufFree(this->partBuffer);
test/src/common/harnessHost.c
+2 -2
View File
@@ -748,8 +748,8 @@ hrnHostConfig(HrnHost *const this)
this->pub.repo1Storage = storageS3New(
hrnHostRepo1Path(this), true, NULL, STRDEF(HRN_HOST_S3_BUCKET), STRDEF(HRN_HOST_S3_ENDPOINT),
storageS3UriStyleHost, STR(HRN_HOST_S3_REGION), storageS3KeyTypeShared, STRDEF(HRN_HOST_S3_ACCESS_KEY),
STRDEF(HRN_HOST_S3_ACCESS_SECRET_KEY), NULL, NULL, NULL, NULL, 5 * 1024 * 1024, NULL, hrnHostIp(s3),
443, ioTimeoutMs(), false, NULL, NULL);
STRDEF(HRN_HOST_S3_ACCESS_SECRET_KEY), NULL, NULL, NULL, NULL, NULL, 5 * 1024 * 1024, NULL,
hrnHostIp(s3), 443, ioTimeoutMs(), false, NULL, NULL);
}
MEM_CONTEXT_OBJ_END();
test/src/module/command/helpTest.c
+1
View File
@@ -321,6 +321,7 @@ testRun(void)
" --repo-s3-kms-key-id S3 repository KMS key\n"
" --repo-s3-region S3 repository region\n"
" --repo-s3-role S3 repository role\n"
" --repo-s3-sse-customer-key S3 Repository SSE Customer Key\n"
" --repo-s3-token S3 repository security token\n"
" --repo-s3-uri-style S3 URI Style [default=host]\n"
" --repo-sftp-host SFTP repository host\n"
test/src/module/storage/s3Test.c
+49 -13
View File
@@ -29,6 +29,7 @@ typedef struct TestRequestParam
const char *securityToken;
const char *range;
const char *kms;
const char *sseC;
const char *ttl;
const char *token;
const char *tag;
@@ -82,6 +83,14 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
if (param.kms != NULL)
strCatZ(request, ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id");
if (param.sseC != NULL)
{
strCatZ(
request,
";x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key"
";x-amz-server-side-encryption-customer-key-md5");
}
if (param.tag != NULL)
strCatZ(request, ";x-amz-tagging");
@@ -136,6 +145,16 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
strCatFmt(request, "x-amz-server-side-encryption-aws-kms-key-id:%s\r\n", param.kms);
}
// Add sseC key
if (param.sseC != NULL)
{
strCatZ(request, "x-amz-server-side-encryption-customer-algorithm:AES256\r\n");
strCatFmt(request, "x-amz-server-side-encryption-customer-key:%s\r\n", param.sseC);
strCatFmt(
request, "x-amz-server-side-encryption-customer-key-md5:%s\r\n",
strZ(strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, bufNewDecode(encodingBase64, STR(param.sseC))))));
}
// Add tags
if (param.tag != NULL)
strCatFmt(request, "x-amz-tagging:%s\r\n", param.tag);
@@ -492,9 +511,11 @@ testRun(void)
hrnCfgArgRaw(argList, cfgOptRepoS3Role, credRole);
hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeAuto);
hrnCfgArgRawZ(argList, cfgOptRepoS3KmsKeyId, "kmskey1");
hrnCfgEnvRawZ(cfgOptRepoS3SseCustomerKey, "rA1P");
hrnCfgArgRawZ(argList, cfgOptRepoStorageTag, "Key1=Value1");
hrnCfgArgRawZ(argList, cfgOptRepoStorageTag, " Key 2= Value 2");
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
hrnCfgEnvRemoveRaw(cfgOptRepoS3SseCustomerKey);
s3 = storageRepoGet(0, true);
driver = (StorageS3 *)storageDriver(s3);
@@ -660,7 +681,7 @@ testRun(void)
hrnServerScriptClose(auth);
testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .accessKey = "x", .securityToken = "z");
testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .accessKey = "x", .securityToken = "z", .sseC = "rA1P");
testResponseP(service, .code = 303, .content = "CONTENT");
StorageRead *read = NULL;
@@ -680,6 +701,9 @@ testRun(void)
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date: <redacted>\n"
"x-amz-security-token: <redacted>\n"
"x-amz-server-side-encryption-customer-algorithm: AES256\n"
"x-amz-server-side-encryption-customer-key: <redacted>\n"
"x-amz-server-side-encryption-customer-key-md5: <redacted>\n"
"*** Response Headers ***:\n"
"content-length: 7\n"
"*** Response Content ***:\n"
@@ -713,7 +737,7 @@ testRun(void)
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz",
.kms = "kmskey1", .tag = "%20Key%202=%20Value%202&Key1=Value1");
.kms = "kmskey1", .sseC = "rA1P", .tag = "%20Key%202=%20Value%202&Key1=Value1");
testResponseP(service);
// Make a copy of the signing key to verify that it gets changed when the keys are updated
@@ -746,7 +770,7 @@ testRun(void)
TEST_TITLE("write zero-length file");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt", .content = "", .kms = "kmskey1",
service, s3, HTTP_VERB_PUT, "/file.txt", .content = "", .kms = "kmskey1", .sseC = "rA1P",
.tag = "%20Key%202=%20Value%202&Key1=Value1");
testResponseP(service);
@@ -757,7 +781,7 @@ testRun(void)
TEST_TITLE("write file in chunks with nothing left over on close");
testRequestP(
service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1",
service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P",
.tag = "%20Key%202=%20Value%202&Key1=Value1");
testResponseP(
service,
@@ -769,10 +793,14 @@ testRun(void)
"<UploadId>WxRt</UploadId>"
"</InitiateMultipartUploadResult>");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456",
.sseC = "rA1P");
testResponseP(service, .header = "etag:WxRt1");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012",
.sseC = "rA1P");
testResponseP(service, .header = "eTag:WxRt2");
testRequestP(
@@ -798,7 +826,7 @@ testRun(void)
// Stop writing tags
driver->tag = NULL;
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P");
testResponseP(
service,
.content =
@@ -809,10 +837,14 @@ testRun(void)
"<UploadId>WxRt</UploadId>"
"</InitiateMultipartUploadResult>");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456",
.sseC = "rA1P");
testResponseP(service, .header = "etag:WxRt1");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012",
.sseC = "rA1P");
testResponseP(service, .header = "eTag:WxRt2");
testRequestP(
@@ -852,7 +884,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with something left over on close");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P");
testResponseP(
service,
.content =
@@ -863,10 +895,14 @@ testRun(void)
"<UploadId>RR55</UploadId>"
"</InitiateMultipartUploadResult>");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456",
.sseC = "rA1P");
testResponseP(service, .header = "etag:RR551");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890",
.sseC = "rA1P");
testResponseP(service, .header = "eTag:RR552");
testRequestP(
@@ -889,7 +925,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("file missing");
testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS");
testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS", .sseC = "rA1P");
testResponseP(service, .code = 404);
TEST_RESULT_BOOL(storageExistsP(s3, STRDEF("BOGUS")), false, "check");