db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-06-20 01:17:49 +02:00
Code Issues Releases Activity

Add support for GCS requester pays.

Browse Source 
GCS user projects allow the bucket requester to be billed rather than the bucket owner. This is useful for hosted services that create backups and want to provide direct (probably read-only) access to the backups.
This commit is contained in:
Timothée Peignier
2025-02-25 14:37:22 -08:00
committed by GitHub
parent 4db0263fd6
commit 814bf487c3
11 changed files with 144 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/xml/release/2025/2.55.0.xml
+7 -2
View File
@@ -51,14 +51,19 @@
</release-item>
<release-item>
<github-pull-request id="2560"/>
<commit subject="Add support for S3 requester pays.">
<github-pull-request id="2560"/>
</commit>
<commit subject="Add support for GCS requester pays.">
<github-pull-request id="2557"/>
</commit>
<release-item-contributor-list>
<release-item-contributor id="timothee.peignier"/>
<release-item-reviewer id="david.steele"/>
</release-item-contributor-list>
<p>Add support for S3 requester pays.</p>
<p>Add support for S3/GCS requester pays.</p>
</release-item>
</release-feature-list>
src/build/config/config.yaml
+7
View File
@@ -2081,6 +2081,13 @@ option:
list:
- gcs
repo-gcs-user-project:
section: global
type: string
group: repo
required: false
command: repo-type
repo-hardlink:
section: global
group: repo
src/build/help/help.xml
+10
View File
@@ -718,6 +718,16 @@
<example>auto</example>
</config-key>
<config-key id="repo-gcs-user-project" name="GCS Repository Project ID">
<summary>GCS project ID.</summary>
<text>
<p>GCS project ID used to determine request billing.</p>
</text>
<example>my-project</example>
</config-key>
<config-key id="repo-host-type" name="Repository Host Protocol Type">
<summary>Repository host protocol type.</summary>
src/config/config.auto.h
+2 -1
View File
@@ -140,7 +140,7 @@ Option constants
#define CFGOPT_VERBOSE "verbose"
#define CFGOPT_VERSION "version"
#define CFG_OPTION_TOTAL 186
#define CFG_OPTION_TOTAL 187
/***********************************************************************************************************************************
Option value constants
@@ -499,6 +499,7 @@ typedef enum
cfgOptRepoGcsEndpoint,
cfgOptRepoGcsKey,
cfgOptRepoGcsKeyType,
cfgOptRepoGcsUserProject,
cfgOptRepoHardlink,
cfgOptRepoHost,
cfgOptRepoHostCaFile,
src/config/parse.auto.c.inc
+68
View File
@@ -6257,6 +6257,73 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
), // opt/repo-gcs-key-type
), // opt/repo-gcs-key-type
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-gcs-user-project
( // opt/repo-gcs-user-project
PARSE_RULE_OPTION_NAME("repo-gcs-user-project"), // opt/repo-gcs-user-project
PARSE_RULE_OPTION_TYPE(String), // opt/repo-gcs-user-project
PARSE_RULE_OPTION_RESET(true), // opt/repo-gcs-user-project
PARSE_RULE_OPTION_REQUIRED(false), // opt/repo-gcs-user-project
PARSE_RULE_OPTION_SECTION(Global), // opt/repo-gcs-user-project
PARSE_RULE_OPTION_GROUP_ID(Repo), // opt/repo-gcs-user-project
// opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-gcs-user-project
( // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Annotate) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(ArchiveGet) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(ArchivePush) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Backup) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Check) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Expire) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Info) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Manifest) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(RepoGet) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(RepoLs) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(RepoPut) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(RepoRm) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Restore) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(StanzaCreate) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(StanzaDelete) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(StanzaUpgrade) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Verify) // opt/repo-gcs-user-project
), // opt/repo-gcs-user-project
// opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/repo-gcs-user-project
( // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(ArchiveGet) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(ArchivePush) // opt/repo-gcs-user-project
), // opt/repo-gcs-user-project
// opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/repo-gcs-user-project
( // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(ArchiveGet) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(ArchivePush) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Backup) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Restore) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Verify) // opt/repo-gcs-user-project
), // opt/repo-gcs-user-project
// opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/repo-gcs-user-project
( // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Annotate) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(ArchiveGet) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(ArchivePush) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Backup) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Check) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Expire) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Info) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Manifest) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(RepoGet) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(RepoLs) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(RepoPut) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(RepoRm) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Restore) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(StanzaCreate) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(StanzaDelete) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(StanzaUpgrade) // opt/repo-gcs-user-project
PARSE_RULE_OPTION_COMMAND(Verify) // opt/repo-gcs-user-project
), // opt/repo-gcs-user-project
), // opt/repo-gcs-user-project
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-hardlink
( // opt/repo-hardlink
PARSE_RULE_OPTION_NAME("repo-hardlink"), // opt/repo-hardlink
@@ -11351,6 +11418,7 @@ static const uint8_t optionResolveOrder[] =
cfgOptRepoBundleLimit, // opt-resolve-order
cfgOptRepoBundleSize, // opt-resolve-order
cfgOptRepoCipherType, // opt-resolve-order
cfgOptRepoGcsUserProject, // opt-resolve-order
cfgOptRepoHardlink, // opt-resolve-order
cfgOptRepoLocal, // opt-resolve-order
cfgOptRepoPath, // opt-resolve-order
src/storage/gcs/helper.c
+1 -1
View File
@@ -28,7 +28,7 @@ storageGcsHelper(const unsigned int repoIdx, const bool write, StoragePathExpres
cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx), (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx),
cfgOptionIdxKvNull(cfgOptRepoStorageTag, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx), cfgOptionIdxStrNull(cfgOptRepoGcsUserProject, repoIdx));
FUNCTION_LOG_RETURN(STORAGE, result);
}
src/storage/gcs/storage.c
+13 -2
View File
@@ -49,6 +49,7 @@ STRING_STATIC(GCS_QUERY_PAGE_TOKEN_STR, "pageToken")
STRING_STATIC(GCS_QUERY_PREFIX_STR, "prefix");
STRING_EXTERN(GCS_QUERY_UPLOAD_ID_STR, GCS_QUERY_UPLOAD_ID);
STRING_STATIC(GCS_QUERY_VERSIONS_STR, "versions");
STRING_STATIC(GCS_QUERY_USER_PROJECT_STR, "userProject");
/***********************************************************************************************************************************
JSON tokens
@@ -98,6 +99,7 @@ struct StorageGcs
size_t chunkSize; // Block size for resumable upload
unsigned int deleteMax; // Maximum objects that can be deleted in one request
const Buffer *tag; // Tags to be applied to objects
const String *userProject; // Project ID
StorageGcsKeyType keyType; // Auth key type
const String *key; // Key (value depends on key type)
@@ -496,7 +498,14 @@ storageGcsRequestAsync(StorageGcs *const this, const String *const verb, Storage
content == NULL || bufEmpty(content) ? ZERO_STR : strNewFmt("%zu", bufUsed(content)));
// Make a copy of the query so it can be modified
HttpQuery *const query = httpQueryDupP(param.query, .redactList = this->queryRedactList);
HttpQuery *const query =
this->userProject != NULL && param.query == NULL ?
httpQueryNewP(.redactList = this->queryRedactList) :
httpQueryDupP(param.query, .redactList = this->queryRedactList);
// Add user project
if (this->userProject != NULL)
httpQueryAdd(query, GCS_QUERY_USER_PROJECT_STR, this->userProject);
// Generate authorization header
if (!param.noAuth)
@@ -1180,7 +1189,7 @@ storageGcsNew(
const String *const path, const bool write, const time_t targetTime, StoragePathExpressionCallback pathExpressionFunction,
const String *const bucket, const StorageGcsKeyType keyType, const String *const key, const size_t chunkSize,
const KeyValue *const tag, const String *const endpoint, const TimeMSec timeout, const bool verifyPeer,
const String *const caFile, const String *const caPath)
const String *const caFile, const String *const caPath, const String *const userProject)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@@ -1197,6 +1206,7 @@ storageGcsNew(
FUNCTION_LOG_PARAM(BOOL, verifyPeer);
FUNCTION_LOG_PARAM(STRING, caFile);
FUNCTION_LOG_PARAM(STRING, caPath);
FUNCTION_LOG_PARAM(STRING, userProject);
FUNCTION_LOG_END();
ASSERT(path != NULL);
@@ -1214,6 +1224,7 @@ storageGcsNew(
.keyType = keyType,
.chunkSize = chunkSize,
.deleteMax = STORAGE_GCS_DELETE_MAX,
.userProject = strDup(userProject),
};
// Create tag JSON buffer
src/storage/gcs/storage.h
+1 -1
View File
@@ -27,6 +27,6 @@ Constructors
FN_EXTERN Storage *storageGcsNew(
const String *path, bool write, time_t targetTime, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
StorageGcsKeyType keyType, const String *key, size_t blockSize, const KeyValue *tag, const String *endpoint, TimeMSec timeout,
bool verifyPeer, const String *caFile, const String *caPath);
bool verifyPeer, const String *caFile, const String *caPath, const String *userProject);
#endif
test/src/common/harnessHost.c
+1 -1
View File
@@ -764,7 +764,7 @@ hrnHostConfig(HrnHost *const this)
this->pub.repo1Storage = storageGcsNew(
hrnHostRepo1Path(this), true, 0, NULL, STRDEF(HRN_HOST_GCS_BUCKET), storageGcsKeyTypeToken,
STRDEF(HRN_HOST_GCS_KEY), 4 * 1024 * 1024, NULL,
strNewFmt("%s:%d", strZ(hrnHostIp(gcs)), HRN_HOST_GCS_PORT), ioTimeoutMs(), false, NULL, NULL);
strNewFmt("%s:%d", strZ(hrnHostIp(gcs)), HRN_HOST_GCS_PORT), ioTimeoutMs(), false, NULL, NULL, NULL);
}
MEM_CONTEXT_OBJ_END();
test/src/module/command/helpTest.c
+1
View File
@@ -316,6 +316,7 @@ testRun(void)
" [default=storage.googleapis.com]\n"
" --repo-gcs-key GCS repository key\n"
" --repo-gcs-key-type GCS repository key type [default=service]\n"
" --repo-gcs-user-project GCS project ID\n"
" --repo-host repository host when operating remotely\n"
" [current=<multi>]\n"
" --repo-host-ca-file repository host certificate authority file\n"
test/src/module/storage/gcsTest.c
+33 -2
View File
@@ -260,7 +260,7 @@ testRun(void)
(StorageGcs *)storageDriver(
storageGcsNew(
STRDEF("/repo"), false, 0, NULL, TEST_BUCKET_STR, storageGcsKeyTypeService, TEST_KEY_FILE_STR, TEST_CHUNK_SIZE,
NULL, TEST_ENDPOINT_STR, TEST_TIMEOUT, true, NULL, NULL)),
NULL, TEST_ENDPOINT_STR, TEST_TIMEOUT, true, NULL, NULL, NULL)),
"read-only gcs storage - service key");
TEST_RESULT_STR_Z(httpUrlHost(storage->authUrl), "test.com", "check host");
TEST_RESULT_STR_Z(httpUrlPath(storage->authUrl), "/token", "check path");
@@ -285,7 +285,7 @@ testRun(void)
(StorageGcs *)storageDriver(
storageGcsNew(
STRDEF("/repo"), true, 0, NULL, TEST_BUCKET_STR, storageGcsKeyTypeService, TEST_KEY_FILE_STR, TEST_CHUNK_SIZE,
NULL, TEST_ENDPOINT_STR, TEST_TIMEOUT, true, NULL, NULL)),
NULL, TEST_ENDPOINT_STR, TEST_TIMEOUT, true, NULL, NULL, NULL)),
"read/write gcs storage - service key");
TEST_RESULT_STR_Z(
@@ -1114,6 +1114,37 @@ testRun(void)
TEST_RESULT_VOID(storagePathRemoveP(storage, STRDEF("/path"), .recurse = true), "remove");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("switch to user project");
hrnServerScriptClose(service);
StringList *argListUserProject = strLstDup(argList);
hrnCfgArgRawZ(argListUserProject, cfgOptRepoGcsUserProject, "usrprj");
hrnCfgArgRawZ(argListUserProject, cfgOptRepoRetentionFull, "1");
HRN_CFG_LOAD(cfgCmdExpire, argListUserProject);
TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage");
hrnServerScriptAccept(service);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("remove file");
testRequestP(service, HTTP_VERB_DELETE, .object = "path/to/test.txt", .query = "userProject=usrprj");
testResponseP(service);
TEST_RESULT_VOID(storageRemoveP(storage, STRDEF("/path/to/test.txt")), "remove");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("info for missing file");
testRequestP(service, HTTP_VERB_GET, .object = "BOGUS", .query = "fields=size%2Cupdated&userProject=usrprj");
testResponseP(service, .code = 404);
TEST_RESULT_BOOL(
storageInfoP(storage, STRDEF("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("switch to target time");