db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-06-20 01:17:49 +02:00
Code Issues Releases Activity

Add support for S3 requester pays.

Browse Source 
S3 requester pays allows the bucket requester to be billed rather than the bucket owner. This is useful for hosted services that create backups and want to provide direct (probably read-only) access to the backups.
This commit is contained in:
Timothée Peignier
2025-02-25 14:12:28 -08:00
committed by GitHub
parent 7941d6506e
commit 4db0263fd6
11 changed files with 151 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/xml/release/2025/2.55.0.xml
+11
View File
@@ -49,6 +49,17 @@
<p>Verify recovery target timeline.</p>
</release-item>
<release-item>
<github-pull-request id="2560"/>
<release-item-contributor-list>
<release-item-contributor id="timothee.peignier"/>
<release-item-reviewer id="david.steele"/>
</release-item-contributor-list>
<p>Add support for S3 requester pays.</p>
</release-item>
</release-feature-list>
<release-improvement-list>
src/build/config/config.yaml
+11
View File
@@ -2410,6 +2410,17 @@ option:
required: false
command: repo-type
repo-s3-requester-pays:
section: global
group: repo
type: boolean
default: false
command: repo-type
depend:
option: repo-type
list:
- s3
repo-s3-uri-style:
section: global
group: repo
src/build/help/help.xml
+10
View File
@@ -1054,6 +1054,16 @@
<example>bceb4f13-6939-4be3-910d-df54dee817b7</example>
</config-key>
<config-key id="repo-s3-requester-pays" name="S3 Repository Requestor Pays">
<summary>S3 repository requester pays.</summary>
<text>
<p>Enables S3 requester pays.</p>
</text>
<example>n</example>
</config-key>
<config-key id="repo-s3-uri-style" name="S3 Repository URI Style">
<summary>S3 URI Style.</summary>
src/config/config.auto.h
+2 -1
View File
@@ -140,7 +140,7 @@ Option constants
#define CFGOPT_VERBOSE "verbose"
#define CFGOPT_VERSION "version"
#define CFG_OPTION_TOTAL 185
#define CFG_OPTION_TOTAL 186
/***********************************************************************************************************************************
Option value constants
@@ -527,6 +527,7 @@ typedef enum
cfgOptRepoS3KeyType,
cfgOptRepoS3KmsKeyId,
cfgOptRepoS3Region,
cfgOptRepoS3RequesterPays,
cfgOptRepoS3Role,
cfgOptRepoS3SseCustomerKey,
cfgOptRepoS3Token,
src/config/parse.auto.c.inc
+86
View File
@@ -7906,6 +7906,91 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
), // opt/repo-s3-region
), // opt/repo-s3-region
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_NAME("repo-s3-requester-pays"), // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_TYPE(Boolean), // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_NEGATE(true), // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_RESET(true), // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_REQUIRED(true), // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_SECTION(Global), // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_GROUP_ID(Repo), // opt/repo-s3-requester-pays
// opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Annotate) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(ArchiveGet) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(ArchivePush) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Backup) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Check) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Expire) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Info) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Manifest) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(RepoGet) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(RepoLs) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(RepoPut) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(RepoRm) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Restore) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(StanzaCreate) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(StanzaDelete) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(StanzaUpgrade) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Verify) // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
// opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(ArchiveGet) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(ArchivePush) // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
// opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(ArchiveGet) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(ArchivePush) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Backup) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Restore) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Verify) // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
// opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Annotate) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(ArchiveGet) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(ArchivePush) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Backup) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Check) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Expire) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Info) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Manifest) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(RepoGet) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(RepoLs) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(RepoPut) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(RepoRm) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Restore) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(StanzaCreate) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(StanzaDelete) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(StanzaUpgrade) // opt/repo-s3-requester-pays
PARSE_RULE_OPTION_COMMAND(Verify) // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
// opt/repo-s3-requester-pays
PARSE_RULE_OPTIONAL // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_OPTIONAL_GROUP // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_OPTIONAL_DEPEND // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_VAL_OPT(RepoType), // opt/repo-s3-requester-pays
PARSE_RULE_VAL_STRID(S3), // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
// opt/repo-s3-requester-pays
PARSE_RULE_OPTIONAL_DEFAULT // opt/repo-s3-requester-pays
( // opt/repo-s3-requester-pays
PARSE_RULE_VAL_BOOL_FALSE, // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
), // opt/repo-s3-requester-pays
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-s3-role
( // opt/repo-s3-role
PARSE_RULE_OPTION_NAME("repo-s3-role"), // opt/repo-s3-role
@@ -11342,6 +11427,7 @@ static const uint8_t optionResolveOrder[] =
cfgOptRepoS3KeyType, // opt-resolve-order
cfgOptRepoS3KmsKeyId, // opt-resolve-order
cfgOptRepoS3Region, // opt-resolve-order
cfgOptRepoS3RequesterPays, // opt-resolve-order
cfgOptRepoS3Role, // opt-resolve-order
cfgOptRepoS3SseCustomerKey, // opt-resolve-order
cfgOptRepoS3Token, // opt-resolve-order
src/storage/s3/helper.c
+1 -1
View File
@@ -91,7 +91,7 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx),
cfgOptionIdxKvNull(cfgOptRepoStorageTag, repoIdx), host, port, ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx), cfgOptionIdxBool(cfgOptRepoS3RequesterPays, repoIdx));
}
MEM_CONTEXT_PRIOR_END();
}
src/storage/s3/storage.c
+10 -1
View File
@@ -39,6 +39,8 @@ STRING_STATIC(S3_HEADER_SSECUSTKEY_AES256_STR, "AES256");
STRING_STATIC(S3_HEADER_SSECUSTKEY_KEY_STR, "x-amz-server-side-encryption-customer-key");
STRING_STATIC(S3_HEADER_SSECUSTKEY_KEY_MD5_STR, "x-amz-server-side-encryption-customer-key-md5");
STRING_STATIC(S3_HEADER_TAGGING, "x-amz-tagging");
STRING_STATIC(S3_HEADER_REQUEST_PAYER, "x-amz-request-payer");
STRING_STATIC(S3_HEADER_REQUEST_PAYER_STR, "requester");
/***********************************************************************************************************************************
S3 query tokens
@@ -110,6 +112,7 @@ struct StorageS3
unsigned int deleteMax; // Maximum objects that can be deleted in one request
StorageS3UriStyle uriStyle; // Path or host style URIs
const String *bucketEndpoint; // Set to {bucket}.{endpoint}
bool requesterPays; // Requester pays?
// For retrieving temporary security credentials
HttpClient *credHttpClient; // HTTP client to service credential requests
@@ -478,6 +481,10 @@ storageS3RequestAsync(StorageS3 *const this, const String *const verb, const Str
strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, param.content)));
}
// Set requester pays when requested
if (this->requesterPays)
httpHeaderPut(requestHeader, S3_HEADER_REQUEST_PAYER, S3_HEADER_REQUEST_PAYER_STR);
// Set KMS headers when requested
if (param.sseKms && this->kmsKeyId != NULL)
{
@@ -1178,7 +1185,8 @@ storageS3New(
const StorageS3KeyType keyType, const String *const accessKey, const String *const secretAccessKey,
const String *const securityToken, const String *const kmsKeyId, const String *sseCustomerKey, const String *const credRole,
const String *const webIdTokenFile, const size_t partSize, const KeyValue *const tag, const String *host,
const unsigned int port, const TimeMSec timeout, const bool verifyPeer, const String *const caFile, const String *const caPath)
const unsigned int port, const TimeMSec timeout, const bool verifyPeer, const String *const caFile, const String *const caPath,
const bool requesterPays)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@@ -1222,6 +1230,7 @@ storageS3New(
.region = strDup(region),
.keyType = keyType,
.kmsKeyId = strDup(kmsKeyId),
.requesterPays = requesterPays,
.sseCustomerKey = strDup(sseCustomerKey),
.partSize = partSize,
.deleteMax = STORAGE_S3_DELETE_MAX,
src/storage/s3/storage.h
+2 -2
View File
@@ -37,7 +37,7 @@ FN_EXTERN Storage *storageS3New(
const String *path, bool write, time_t targetTime, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
const String *secretAccessKey, const String *securityToken, const String *kmsKeyId, const String *sseCustomerKey,
const String *credRole, const String *webIdTokenFile, size_t partSize, const KeyValue *tag, const String *host, unsigned int port,
TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
const String *credRole, const String *webIdTokenFile, size_t partSize, const KeyValue *tag, const String *host,
unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath, bool requesterPays);
#endif
test/src/common/harnessHost.c
+1 -1
View File
@@ -788,7 +788,7 @@ hrnHostConfig(HrnHost *const this)
hrnHostRepo1Path(this), true, 0, NULL, STRDEF(HRN_HOST_S3_BUCKET), STRDEF(HRN_HOST_S3_ENDPOINT),
storageS3UriStyleHost, STR(HRN_HOST_S3_REGION), storageS3KeyTypeShared, STRDEF(HRN_HOST_S3_ACCESS_KEY),
STRDEF(HRN_HOST_S3_ACCESS_SECRET_KEY), NULL, NULL, NULL, NULL, NULL, 5 * 1024 * 1024, NULL,
hrnHostIp(s3), 443, ioTimeoutMs(), false, NULL, NULL);
hrnHostIp(s3), 443, ioTimeoutMs(), false, NULL, NULL, NULL);
}
MEM_CONTEXT_OBJ_END();
test/src/module/command/helpTest.c
+1
View File
@@ -346,6 +346,7 @@ testRun(void)
" --repo-s3-key-type S3 repository key type [default=shared]\n"
" --repo-s3-kms-key-id S3 repository KMS key\n"
" --repo-s3-region S3 repository region\n"
" --repo-s3-requester-pays S3 repository requester pays [default=n]\n"
" --repo-s3-role S3 repository role\n"
" --repo-s3-sse-customer-key S3 repository SSE customer key\n"
" --repo-s3-token S3 repository security token\n"
test/src/module/storage/s3Test.c
+16 -5
View File
@@ -33,6 +33,7 @@ typedef struct TestRequestParam
const char *ttl;
const char *token;
const char *tag;
bool requesterPays;
} TestRequestParam;
#define testRequestP(write, s3, verb, path, ...) \
@@ -77,6 +78,9 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
strCatZ(request, "x-amz-content-sha256;x-amz-date");
if (param.requesterPays)
strCatZ(request, ";x-amz-request-payer");
if (securityToken != NULL)
strCatZ(request, ";x-amz-security-token");
@@ -138,6 +142,10 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
strCatFmt(request, "x-amz-security-token:%s\r\n", securityToken);
}
// Requestor pays
if (param.requesterPays)
strCatZ(request, "x-amz-request-payer:requester\r\n");
// Add kms key
if (param.kms != NULL)
{
@@ -1447,6 +1455,7 @@ testRun(void)
hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg1");
hrnCfgArgRawZ(argList, cfgOptRepo, "1");
hrnCfgArgRawZ(argList, cfgOptRepoTargetTime, "2024-08-04 02:54:09+00");
hrnCfgArgRawBool(argList, cfgOptRepoS3RequesterPays, true);
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
s3 = storageRepoGet(0, false);
@@ -1456,7 +1465,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("list with time limit");
testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&prefix=path%2Fto%2F&versions=");
testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&prefix=path%2Fto%2F&versions=", .requesterPays = true);
testResponseP(
service,
.content =
@@ -1474,7 +1483,9 @@ testRun(void)
" </Version>"
"</ListBucketResult>");
testRequestP(service, s3, HTTP_VERB_GET, "/?continuation-token=1ueG&delimiter=%2F&prefix=path%2Fto%2F&versions=");
testRequestP(
service, s3, HTTP_VERB_GET, "/?continuation-token=1ueG&delimiter=%2F&prefix=path%2Fto%2F&versions=",
.requesterPays = true);
testResponseP(
service,
.content =
@@ -1509,7 +1520,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with time limit");
testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&prefix=path%2F3%2F&versions=");
testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&prefix=path%2F3%2F&versions=", .requesterPays = true);
testResponseP(
service,
.content =
@@ -1531,7 +1542,7 @@ testRun(void)
" </CommonPrefixes>"
"</ListBucketResult>");
testRequestP(service, s3, HTTP_VERB_GET, "/path/3/test_file?versionId=bbbb");
testRequestP(service, s3, HTTP_VERB_GET, "/path/3/test_file?versionId=bbbb", .requesterPays = true);
testResponseP(service, .content = "123456");
TEST_RESULT_STR_Z(
@@ -1540,7 +1551,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get missing file with time limit");
testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&versions=");
testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&versions=", .requesterPays = true);
testResponseP(
service,
.content =