db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Add automatic GCS authentication for GCE instances.

Browse Source 
When running on a GCE instance the authentication token can be pulled directly from the instance metadata. This is configured with repo-gcs-key-type=auto.

In a separate commit (26fefa6), move the code that parses the token response into a separate function, storageGcsAuthToken(), since it is now needed by two key types. This drastically improves the readability of the main commit.
This commit is contained in:
David Steele 2021-05-17 14:55:50 -04:00 committed by GitHub
parent 0152075e6b
commit 9af033194a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 171 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/xml/reference.xml
View File

@ -433,6 +433,7 @@
<text>The following types are supported for authorization:
<ul>
<li><id>auto</id> - Authorize using the instance service account.</li>
<li><id>service</id> - Service account from locally stored key.</li>
<li><id>token</id> - For local testing, e.g. <file>fakegcs</file>.</li>
</ul></text>

24
doc/xml/release.xml
View File

@ -57,6 +57,23 @@
</release-bug-list>
<release-feature-list>
<release-item>
<commit subject="Use existing variable for GCS test server port."/>
<commit subject="Refactor GCS token response parsing into a separate function."/>
<commit subject="Add automatic GCS authentication for GCE instances.">
<github-pull-request id="1395"/>
</commit>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="jan.wieck"/>
<!-- Actually tester, but we don't have a tag for that yet -->
<release-item-reviewer id="daniel.farina"/>
</release-item-contributor-list>
<p>Add automatic <proper>GCS</proper> authentication for <proper>GCE</proper> instances.</p>
</release-item>
<release-item>
<github-issue id="986"/>
<github-pull-request id="1337"/>
@ -276,6 +293,8 @@
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>
<!-- Actually tester, but we don't have a tag for that yet -->
<release-item-reviewer id="daniel.farina"/>
</release-item-contributor-list>
<p>GCS support for repository storage.</p>
@ -9575,6 +9594,11 @@
<contributor-id type="github">farrellit</contributor-id>
</contributor>
<contributor id="daniel.farina">
<contributor-name-display>Daniel Farina</contributor-name-display>
<contributor-id type="github">fdr</contributor-id>
</contributor>
<contributor id="daniel.westermann">
<contributor-name-display>Daniel Westermann</contributor-name-display>
<contributor-id type="github">danielwestermann</contributor-id>

2
doc/xml/user-guide.xml
View File

@ -669,6 +669,8 @@
<backrest-config-option section="global" key="process-max">4</backrest-config-option>
</backrest-config>
<p>When running in <proper>GCE</proper> set <br-option>repo{[gcs-setup-repo-id]}-gcs-key-type=auto</br-option> to automatically authenticate using the instance service account.</p>
</block-define>
<!-- ======================================================================================================================= -->

7
src/build/config/config.yaml
View File

@ -1536,7 +1536,11 @@ option:
group: repo
secure: true
command: repo-type
depend: repo-gcs-bucket
depend:
option: repo-gcs-key-type
list:
- service
- token
repo-gcs-key-type:
section: global
@ -1544,6 +1548,7 @@ option:
group: repo
default: service
allow-list:
- auto
- service
- token
command: repo-type

5
src/command/help/help.auto.c
View File

@ -2907,10 +2907,13 @@ static const unsigned char helpDataPack[] =
pckTypeStr << 4 | 0x08, 0x18, // Summary
0x47, 0x43, 0x53, 0x20, 0x72, 0x65, 0x70, 0x6F, 0x73, 0x69, 0x74, 0x6F, 0x72, 0x79, 0x20, 0x6B, 0x65, 0x79, 0x20, 0x74,
0x79, 0x70, 0x65, 0x2E,
pckTypeStr << 4 | 0x08, 0x95, 0x01, // Description
pckTypeStr << 4 | 0x08, 0xCC, 0x01, // Description
0x54, 0x68, 0x65, 0x20, 0x66, 0x6F, 0x6C, 0x6C, 0x6F, 0x77, 0x69, 0x6E, 0x67, 0x20, 0x74, 0x79, 0x70, 0x65, 0x73, 0x20,
0x61, 0x72, 0x65, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6F, 0x72, 0x74, 0x65, 0x64, 0x20, 0x66, 0x6F, 0x72, 0x20, 0x61, 0x75,
0x74, 0x68, 0x6F, 0x72, 0x69, 0x7A, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x3A, 0x0A, 0x0A,
0x2A, 0x20, 0x61, 0x75, 0x74, 0x6F, 0x20, 0x2D, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6F, 0x72, 0x69, 0x7A, 0x65, 0x20, 0x75,
0x73, 0x69, 0x6E, 0x67, 0x20, 0x74, 0x68, 0x65, 0x20, 0x69, 0x6E, 0x73, 0x74, 0x61, 0x6E, 0x63, 0x65, 0x20, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x61, 0x63, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x2E, 0x0A,
0x2A, 0x20, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x2D, 0x20, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20,
0x61, 0x63, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x20, 0x66, 0x72, 0x6F, 0x6D, 0x20, 0x6C, 0x6F, 0x63, 0x61, 0x6C, 0x6C, 0x79,
0x20, 0x73, 0x74, 0x6F, 0x72, 0x65, 0x64, 0x20, 0x6B, 0x65, 0x79, 0x2E, 0x0A,

1
src/config/config.auto.h
View File

@ -187,6 +187,7 @@ Option value constants
#define CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC_Z "aes-256-cbc"
#define CFGOPTVAL_REPO_CIPHER_TYPE_NONE_Z "none"
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_AUTO_Z "auto"
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_SERVICE_Z "service"
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_TOKEN_Z "token"

8
src/config/parse.auto.c
View File

@ -3795,7 +3795,12 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
PARSE_RULE_OPTION_OPTIONAL_LIST
(
PARSE_RULE_OPTION_OPTIONAL_DEPEND(cfgOptRepoGcsKeyType),
PARSE_RULE_OPTION_OPTIONAL_DEPEND_LIST
(
cfgOptRepoGcsKeyType,
"service",
"token"
),
),
),
@ -3866,6 +3871,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
(
PARSE_RULE_OPTION_OPTIONAL_ALLOW_LIST
(
"auto",
"service",
"token"
),

94
src/storage/gcs/storage.c
View File

@ -31,6 +31,8 @@ GCS Storage
HTTP headers
***********************************************************************************************************************************/
STRING_EXTERN(GCS_HEADER_UPLOAD_ID_STR, GCS_HEADER_UPLOAD_ID);
STRING_STATIC(GCS_HEADER_METADATA_FLAVOR_STR, "metadata-flavor");
STRING_STATIC(GCS_HEADER_GOOGLE_STR, "Google");
/***********************************************************************************************************************************
Query tokens
@ -269,6 +271,41 @@ storageGcsAuthService(StorageGcs *this, time_t timeBegin)
FUNCTION_TEST_RETURN(result);
}
/***********************************************************************************************************************************
Get authentication token automatically for instances running in GCE.
Based on the documentation at https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#applications
***********************************************************************************************************************************/
static StorageGcsAuthTokenResult
storageGcsAuthAuto(StorageGcs *this, time_t timeBegin)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_GCS, this);
FUNCTION_TEST_PARAM(TIME, timeBegin);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(timeBegin > 0);
StorageGcsAuthTokenResult result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
HttpHeader *header = httpHeaderNew(NULL);
httpHeaderAdd(header, HTTP_HEADER_HOST_STR, httpUrlHost(this->authUrl));
httpHeaderAdd(header, GCS_HEADER_METADATA_FLAVOR_STR, GCS_HEADER_GOOGLE_STR);
httpHeaderAdd(header, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
HttpRequest *request = httpRequestNewP(
this->authClient, HTTP_VERB_GET_STR, httpUrlPath(this->authUrl), NULL, .header = header);
result = storageGcsAuthToken(request, timeBegin);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN(result);
}
/***********************************************************************************************************************************
Generate authorization header and add it to the supplied header list
***********************************************************************************************************************************/
@ -286,37 +323,34 @@ storageGcsAuth(StorageGcs *this, HttpHeader *httpHeader)
MEM_CONTEXT_TEMP_BEGIN()
{
// Process authentication type
switch (this->keyType)
// Get the token if it was not supplied by the user
if (this->keyType != storageGcsKeyTypeToken)
{
// Service key authentication requests a token then drops through to normal token authentication
case storageGcsKeyTypeService:
ASSERT(this->keyType == storageGcsKeyTypeAuto || this->keyType == storageGcsKeyTypeService);
time_t timeBegin = time(NULL);
// If the current token has expired then request a new one
if (timeBegin >= this->tokenTimeExpire)
{
time_t timeBegin = time(NULL);
StorageGcsAuthTokenResult tokenResult = this->keyType == storageGcsKeyTypeAuto ?
storageGcsAuthAuto(this, timeBegin) : storageGcsAuthService(this, timeBegin);
// If the current token has expired then request a new one
if (timeBegin >= this->tokenTimeExpire)
MEM_CONTEXT_BEGIN(this->memContext)
{
StorageGcsAuthTokenResult tokenResult = storageGcsAuthService(this, timeBegin);
strFree(this->token);
this->token = strNewFmt("%s %s", strZ(tokenResult.tokenType), strZ(tokenResult.token));
MEM_CONTEXT_BEGIN(this->memContext)
{
strFree(this->token);
this->token = strNewFmt("%s %s", strZ(tokenResult.tokenType), strZ(tokenResult.token));
// Subtract http client timeout * 2 so the token does not expire in the middle of http retries
this->tokenTimeExpire =
tokenResult.timeExpire - ((time_t)(httpClientTimeout(this->httpClient) / MSEC_PER_SEC * 2));
}
MEM_CONTEXT_END();
// Subtract http client timeout * 2 so the token does not expire in the middle of http retries
this->tokenTimeExpire =
tokenResult.timeExpire - ((time_t)(httpClientTimeout(this->httpClient) / MSEC_PER_SEC * 2));
}
MEM_CONTEXT_END();
}
// Token authentication
case storageGcsKeyTypeToken:
httpHeaderPut(httpHeader, HTTP_HEADER_AUTHORIZATION_STR, this->token);
break;
}
// Add authorization header
httpHeaderPut(httpHeader, HTTP_HEADER_AUTHORIZATION_STR, this->token);
}
MEM_CONTEXT_TEMP_END();
@ -895,7 +929,7 @@ storageGcsNew(
ASSERT(path != NULL);
ASSERT(bucket != NULL);
ASSERT(key != NULL);
ASSERT(keyType == storageGcsKeyTypeAuto || key != NULL);
ASSERT(chunkSize != 0);
Storage *this = NULL;
@ -917,6 +951,18 @@ storageGcsNew(
// Handle auth key types
switch (keyType)
{
// Auto authentication for GCE instances
case storageGcsKeyTypeAuto:
{
driver->authUrl = httpUrlNewParseP(
STRDEF("metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"),
.type = httpProtocolTypeHttp);
driver->authClient = httpClientNew(
sckClientNew(httpUrlHost(driver->authUrl), httpUrlPort(driver->authUrl), timeout), timeout);
break;
}
// Read data from file for service keys
case storageGcsKeyTypeService:
{

1
src/storage/gcs/storage.h
View File

@ -16,6 +16,7 @@ Key type
***********************************************************************************************************************************/
typedef enum
{
storageGcsKeyTypeAuto = STRID5("auto", 0x7d2a10),
storageGcsKeyTypeService = STRID5("service", 0x1469b48b30),
storageGcsKeyTypeToken = STRID5("token", 0xe2adf40),
} StorageGcsKeyType;

5
src/storage/helper.c
View File

@ -381,8 +381,9 @@ storageRepoGet(unsigned int repoIdx, bool write)
result = storageGcsNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
(StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsKey, repoIdx),
STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
(StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx), STORAGE_GCS_CHUNKSIZE_DEFAULT,
cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
break;

55
test/src/module/storage/gcsTest.c
View File

@ -190,6 +190,7 @@ testRun(void)
const String *const testHost = hrnServerHost();
const unsigned int testPort = hrnServerPort(0);
const unsigned int testPortAuth = hrnServerPort(1);
const unsigned int testPortMeta = hrnServerPort(2);
// *****************************************************************************************************************************
if (testBegin("storageRepoGet()"))
@ -298,12 +299,24 @@ testRun(void)
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_CHILD_BEGIN(0, true)
{
TEST_RESULT_VOID(
hrnServerRunP(
ioFdReadNew(strNew("meta server read"), HARNESS_FORK_CHILD_READ(), 10000), hrnServerProtocolSocket,
.port = testPortMeta),
"meta server run");
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
IoWrite *service = hrnServerScriptBegin(
ioFdWriteNew(strNew("gcs client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000));
IoWrite *auth = hrnServerScriptBegin(
ioFdWriteNew(strNew("auth client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(1), 2000));
IoWrite *meta = hrnServerScriptBegin(
ioFdWriteNew(strNew("meta client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(2), 2000));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("test service auth");
@ -317,13 +330,11 @@ testRun(void)
hrnCfgArgRawBool(argList, cfgOptRepoStorageVerifyTls, testContainer());
hrnCfgEnvRawZ(cfgOptRepoGcsKey, TEST_KEY_FILE);
harnessCfgLoad(cfgCmdArchivePush, argList);
hrnCfgEnvRemoveRaw(cfgOptRepoGcsKey);
Storage *storage = NULL;
TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage");
// Tests need the chunk size to be 16
((StorageGcs *)storageDriver(storage))->chunkSize = 16;
// Generate the auth request. The JWT part will need to be ? since it can vary in content and size.
const char *const preamble = "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=";
const String *const jwt = storageGcsAuthJwt(((StorageGcs *)storageDriver(storage)), time(NULL));
@ -407,9 +418,47 @@ testRun(void)
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(storage, strNew("file.txt")))), "this is a sample file", "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("switch to auto auth");
hrnServerScriptClose(service);
StringList *argListAuto = strLstDup(argList);
hrnCfgArgRawStrId(argListAuto, cfgOptRepoGcsKeyType, storageGcsKeyTypeAuto);
harnessCfgLoad(cfgCmdArchivePush, argListAuto);
TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage");
// Replace the default authClient with one that points locally. The default host and url will still be used so they
// can be verified when testing auth.
((StorageGcs *)storageDriver(storage))->authClient = httpClientNew(
sckClientNew(hrnServerHost(), testPortMeta, 2000), 2000);
// Tests need the chunk size to be 16
((StorageGcs *)storageDriver(storage))->chunkSize = 16;
hrnServerScriptAccept(service);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get zero-length file");
// Get token automatically from metadata
hrnServerScriptAccept(meta);
hrnServerScriptExpectZ(
meta,
"GET /computeMetadata/v1/instance/service-accounts/default/token HTTP/1.1\r\n"
"user-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n"
"content-length:0\r\n"
"host:metadata.google.internal\r\n"
"metadata-flavor:Google\r\n"
"\r\n");
testResponseP(meta, .content = "{\"access_token\":\"X\",\"token_type\":\"X\",\"expires_in\":3600}");
hrnServerScriptClose(meta);
// Meta service no longer needed
hrnServerScriptEnd(meta);
testRequestP(service, HTTP_VERB_GET, .object = "file0.txt", .query = "alt=media");
testResponseP(service);