From 9af033194abc4b1cd3f06c3a97fff71c966c3827 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 17 May 2021 14:55:50 -0400 Subject: [PATCH] Add automatic GCS authentication for GCE instances. When running on a GCE instance the authentication token can be pulled directly from the instance metadata. This is configured with repo-gcs-key-type=auto. In a separate commit (26fefa6), move the code that parses the token response into a separate function, storageGcsAuthToken(), since it is now needed by two key types. This drastically improves the readability of the main commit. --- doc/xml/reference.xml | 1 + doc/xml/release.xml | 24 ++++++++ doc/xml/user-guide.xml | 2 + src/build/config/config.yaml | 7 ++- src/command/help/help.auto.c | 5 +- src/config/config.auto.h | 1 + src/config/parse.auto.c | 8 ++- src/storage/gcs/storage.c | 94 +++++++++++++++++++++++-------- src/storage/gcs/storage.h | 1 + src/storage/helper.c | 5 +- test/src/module/storage/gcsTest.c | 55 +++++++++++++++++- 11 files changed, 171 insertions(+), 32 deletions(-) diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index c9f651bfa..44b91509f 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -433,6 +433,7 @@ The following types are supported for authorization: diff --git a/doc/xml/release.xml b/doc/xml/release.xml index af3a7ba90..af7dc52ca 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -57,6 +57,23 @@ + + + + + + + + + + + + + + +

Add automatic GCS authentication for GCE instances.

+ + @@ -276,6 +293,8 @@ + +

GCS support for repository storage.

@@ -9575,6 +9594,11 @@ farrellit + + Daniel Farina + fdr + + Daniel Westermann danielwestermann diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index 7ec021e4b..0ba8005e0 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -669,6 +669,8 @@ 4 + +

When running in GCE set repo{[gcs-setup-repo-id]}-gcs-key-type=auto to automatically authenticate using the instance service account.

diff --git a/src/build/config/config.yaml b/src/build/config/config.yaml index 677929fb5..8e7038ecf 100644 --- a/src/build/config/config.yaml +++ b/src/build/config/config.yaml @@ -1536,7 +1536,11 @@ option: group: repo secure: true command: repo-type - depend: repo-gcs-bucket + depend: + option: repo-gcs-key-type + list: + - service + - token repo-gcs-key-type: section: global @@ -1544,6 +1548,7 @@ option: group: repo default: service allow-list: + - auto - service - token command: repo-type diff --git a/src/command/help/help.auto.c b/src/command/help/help.auto.c index ddb65f66e..fbf80674b 100644 --- a/src/command/help/help.auto.c +++ b/src/command/help/help.auto.c @@ -2907,10 +2907,13 @@ static const unsigned char helpDataPack[] = pckTypeStr << 4 | 0x08, 0x18, // Summary 0x47, 0x43, 0x53, 0x20, 0x72, 0x65, 0x70, 0x6F, 0x73, 0x69, 0x74, 0x6F, 0x72, 0x79, 0x20, 0x6B, 0x65, 0x79, 0x20, 0x74, 0x79, 0x70, 0x65, 0x2E, - pckTypeStr << 4 | 0x08, 0x95, 0x01, // Description + pckTypeStr << 4 | 0x08, 0xCC, 0x01, // Description 0x54, 0x68, 0x65, 0x20, 0x66, 0x6F, 0x6C, 0x6C, 0x6F, 0x77, 0x69, 0x6E, 0x67, 0x20, 0x74, 0x79, 0x70, 0x65, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6F, 0x72, 0x74, 0x65, 0x64, 0x20, 0x66, 0x6F, 0x72, 0x20, 0x61, 0x75, 0x74, 0x68, 0x6F, 0x72, 0x69, 0x7A, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x3A, 0x0A, 0x0A, + 0x2A, 0x20, 0x61, 0x75, 0x74, 0x6F, 0x20, 0x2D, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6F, 0x72, 0x69, 0x7A, 0x65, 0x20, 0x75, + 0x73, 0x69, 0x6E, 0x67, 0x20, 0x74, 0x68, 0x65, 0x20, 0x69, 0x6E, 0x73, 0x74, 0x61, 0x6E, 0x63, 0x65, 0x20, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x61, 0x63, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x2E, 0x0A, 0x2A, 0x20, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x2D, 0x20, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x61, 0x63, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x20, 0x66, 0x72, 0x6F, 0x6D, 0x20, 0x6C, 0x6F, 0x63, 0x61, 0x6C, 0x6C, 0x79, 0x20, 0x73, 0x74, 0x6F, 0x72, 0x65, 0x64, 0x20, 0x6B, 0x65, 0x79, 0x2E, 0x0A, diff --git a/src/config/config.auto.h b/src/config/config.auto.h index b05370f65..ddd120329 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -187,6 +187,7 @@ Option value constants #define CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC_Z "aes-256-cbc" #define CFGOPTVAL_REPO_CIPHER_TYPE_NONE_Z "none" +#define CFGOPTVAL_REPO_GCS_KEY_TYPE_AUTO_Z "auto" #define CFGOPTVAL_REPO_GCS_KEY_TYPE_SERVICE_Z "service" #define CFGOPTVAL_REPO_GCS_KEY_TYPE_TOKEN_Z "token" diff --git a/src/config/parse.auto.c b/src/config/parse.auto.c index 250a8d641..278b58b5a 100644 --- a/src/config/parse.auto.c +++ b/src/config/parse.auto.c @@ -3795,7 +3795,12 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = PARSE_RULE_OPTION_OPTIONAL_LIST ( - PARSE_RULE_OPTION_OPTIONAL_DEPEND(cfgOptRepoGcsKeyType), + PARSE_RULE_OPTION_OPTIONAL_DEPEND_LIST + ( + cfgOptRepoGcsKeyType, + "service", + "token" + ), ), ), @@ -3866,6 +3871,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = ( PARSE_RULE_OPTION_OPTIONAL_ALLOW_LIST ( + "auto", "service", "token" ), diff --git a/src/storage/gcs/storage.c b/src/storage/gcs/storage.c index 55a848641..2e8e08ce8 100644 --- a/src/storage/gcs/storage.c +++ b/src/storage/gcs/storage.c @@ -31,6 +31,8 @@ GCS Storage HTTP headers ***********************************************************************************************************************************/ STRING_EXTERN(GCS_HEADER_UPLOAD_ID_STR, GCS_HEADER_UPLOAD_ID); +STRING_STATIC(GCS_HEADER_METADATA_FLAVOR_STR, "metadata-flavor"); +STRING_STATIC(GCS_HEADER_GOOGLE_STR, "Google"); /*********************************************************************************************************************************** Query tokens @@ -269,6 +271,41 @@ storageGcsAuthService(StorageGcs *this, time_t timeBegin) FUNCTION_TEST_RETURN(result); } +/*********************************************************************************************************************************** +Get authentication token automatically for instances running in GCE. + +Based on the documentation at https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#applications +***********************************************************************************************************************************/ +static StorageGcsAuthTokenResult +storageGcsAuthAuto(StorageGcs *this, time_t timeBegin) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE_GCS, this); + FUNCTION_TEST_PARAM(TIME, timeBegin); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(timeBegin > 0); + + StorageGcsAuthTokenResult result = {0}; + + MEM_CONTEXT_TEMP_BEGIN() + { + HttpHeader *header = httpHeaderNew(NULL); + httpHeaderAdd(header, HTTP_HEADER_HOST_STR, httpUrlHost(this->authUrl)); + httpHeaderAdd(header, GCS_HEADER_METADATA_FLAVOR_STR, GCS_HEADER_GOOGLE_STR); + httpHeaderAdd(header, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR); + + HttpRequest *request = httpRequestNewP( + this->authClient, HTTP_VERB_GET_STR, httpUrlPath(this->authUrl), NULL, .header = header); + + result = storageGcsAuthToken(request, timeBegin); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_TEST_RETURN(result); +} + /*********************************************************************************************************************************** Generate authorization header and add it to the supplied header list ***********************************************************************************************************************************/ @@ -286,37 +323,34 @@ storageGcsAuth(StorageGcs *this, HttpHeader *httpHeader) MEM_CONTEXT_TEMP_BEGIN() { - // Process authentication type - switch (this->keyType) + // Get the token if it was not supplied by the user + if (this->keyType != storageGcsKeyTypeToken) { - // Service key authentication requests a token then drops through to normal token authentication - case storageGcsKeyTypeService: + ASSERT(this->keyType == storageGcsKeyTypeAuto || this->keyType == storageGcsKeyTypeService); + + time_t timeBegin = time(NULL); + + // If the current token has expired then request a new one + if (timeBegin >= this->tokenTimeExpire) { - time_t timeBegin = time(NULL); + StorageGcsAuthTokenResult tokenResult = this->keyType == storageGcsKeyTypeAuto ? + storageGcsAuthAuto(this, timeBegin) : storageGcsAuthService(this, timeBegin); - // If the current token has expired then request a new one - if (timeBegin >= this->tokenTimeExpire) + MEM_CONTEXT_BEGIN(this->memContext) { - StorageGcsAuthTokenResult tokenResult = storageGcsAuthService(this, timeBegin); + strFree(this->token); + this->token = strNewFmt("%s %s", strZ(tokenResult.tokenType), strZ(tokenResult.token)); - MEM_CONTEXT_BEGIN(this->memContext) - { - strFree(this->token); - this->token = strNewFmt("%s %s", strZ(tokenResult.tokenType), strZ(tokenResult.token)); - - // Subtract http client timeout * 2 so the token does not expire in the middle of http retries - this->tokenTimeExpire = - tokenResult.timeExpire - ((time_t)(httpClientTimeout(this->httpClient) / MSEC_PER_SEC * 2)); - } - MEM_CONTEXT_END(); + // Subtract http client timeout * 2 so the token does not expire in the middle of http retries + this->tokenTimeExpire = + tokenResult.timeExpire - ((time_t)(httpClientTimeout(this->httpClient) / MSEC_PER_SEC * 2)); } + MEM_CONTEXT_END(); } - - // Token authentication - case storageGcsKeyTypeToken: - httpHeaderPut(httpHeader, HTTP_HEADER_AUTHORIZATION_STR, this->token); - break; } + + // Add authorization header + httpHeaderPut(httpHeader, HTTP_HEADER_AUTHORIZATION_STR, this->token); } MEM_CONTEXT_TEMP_END(); @@ -895,7 +929,7 @@ storageGcsNew( ASSERT(path != NULL); ASSERT(bucket != NULL); - ASSERT(key != NULL); + ASSERT(keyType == storageGcsKeyTypeAuto || key != NULL); ASSERT(chunkSize != 0); Storage *this = NULL; @@ -917,6 +951,18 @@ storageGcsNew( // Handle auth key types switch (keyType) { + // Auto authentication for GCE instances + case storageGcsKeyTypeAuto: + { + driver->authUrl = httpUrlNewParseP( + STRDEF("metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"), + .type = httpProtocolTypeHttp); + driver->authClient = httpClientNew( + sckClientNew(httpUrlHost(driver->authUrl), httpUrlPort(driver->authUrl), timeout), timeout); + + break; + } + // Read data from file for service keys case storageGcsKeyTypeService: { diff --git a/src/storage/gcs/storage.h b/src/storage/gcs/storage.h index df23410d1..7d5bdd5a2 100644 --- a/src/storage/gcs/storage.h +++ b/src/storage/gcs/storage.h @@ -16,6 +16,7 @@ Key type ***********************************************************************************************************************************/ typedef enum { + storageGcsKeyTypeAuto = STRID5("auto", 0x7d2a10), storageGcsKeyTypeService = STRID5("service", 0x1469b48b30), storageGcsKeyTypeToken = STRID5("token", 0xe2adf40), } StorageGcsKeyType; diff --git a/src/storage/helper.c b/src/storage/helper.c index e22aee849..1aec9c38e 100644 --- a/src/storage/helper.c +++ b/src/storage/helper.c @@ -381,8 +381,9 @@ storageRepoGet(unsigned int repoIdx, bool write) result = storageGcsNew( cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression, cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx), - (StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsKey, repoIdx), - STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(), + (StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), + cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx), STORAGE_GCS_CHUNKSIZE_DEFAULT, + cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); break; diff --git a/test/src/module/storage/gcsTest.c b/test/src/module/storage/gcsTest.c index 39134aafc..96c5da790 100644 --- a/test/src/module/storage/gcsTest.c +++ b/test/src/module/storage/gcsTest.c @@ -190,6 +190,7 @@ testRun(void) const String *const testHost = hrnServerHost(); const unsigned int testPort = hrnServerPort(0); const unsigned int testPortAuth = hrnServerPort(1); + const unsigned int testPortMeta = hrnServerPort(2); // ***************************************************************************************************************************** if (testBegin("storageRepoGet()")) @@ -298,12 +299,24 @@ testRun(void) } HARNESS_FORK_CHILD_END(); + HARNESS_FORK_CHILD_BEGIN(0, true) + { + TEST_RESULT_VOID( + hrnServerRunP( + ioFdReadNew(strNew("meta server read"), HARNESS_FORK_CHILD_READ(), 10000), hrnServerProtocolSocket, + .port = testPortMeta), + "meta server run"); + } + HARNESS_FORK_CHILD_END(); + HARNESS_FORK_PARENT_BEGIN() { IoWrite *service = hrnServerScriptBegin( ioFdWriteNew(strNew("gcs client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000)); IoWrite *auth = hrnServerScriptBegin( ioFdWriteNew(strNew("auth client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(1), 2000)); + IoWrite *meta = hrnServerScriptBegin( + ioFdWriteNew(strNew("meta client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(2), 2000)); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("test service auth"); @@ -317,13 +330,11 @@ testRun(void) hrnCfgArgRawBool(argList, cfgOptRepoStorageVerifyTls, testContainer()); hrnCfgEnvRawZ(cfgOptRepoGcsKey, TEST_KEY_FILE); harnessCfgLoad(cfgCmdArchivePush, argList); + hrnCfgEnvRemoveRaw(cfgOptRepoGcsKey); Storage *storage = NULL; TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage"); - // Tests need the chunk size to be 16 - ((StorageGcs *)storageDriver(storage))->chunkSize = 16; - // Generate the auth request. The JWT part will need to be ? since it can vary in content and size. const char *const preamble = "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion="; const String *const jwt = storageGcsAuthJwt(((StorageGcs *)storageDriver(storage)), time(NULL)); @@ -407,9 +418,47 @@ testRun(void) TEST_RESULT_STR_Z( strNewBuf(storageGetP(storageNewReadP(storage, strNew("file.txt")))), "this is a sample file", "get file"); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("switch to auto auth"); + + hrnServerScriptClose(service); + + StringList *argListAuto = strLstDup(argList); + hrnCfgArgRawStrId(argListAuto, cfgOptRepoGcsKeyType, storageGcsKeyTypeAuto); + harnessCfgLoad(cfgCmdArchivePush, argListAuto); + + TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage"); + + // Replace the default authClient with one that points locally. The default host and url will still be used so they + // can be verified when testing auth. + ((StorageGcs *)storageDriver(storage))->authClient = httpClientNew( + sckClientNew(hrnServerHost(), testPortMeta, 2000), 2000); + + // Tests need the chunk size to be 16 + ((StorageGcs *)storageDriver(storage))->chunkSize = 16; + + hrnServerScriptAccept(service); + // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("get zero-length file"); + // Get token automatically from metadata + hrnServerScriptAccept(meta); + hrnServerScriptExpectZ( + meta, + "GET /computeMetadata/v1/instance/service-accounts/default/token HTTP/1.1\r\n" + "user-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n" + "content-length:0\r\n" + "host:metadata.google.internal\r\n" + "metadata-flavor:Google\r\n" + "\r\n"); + + testResponseP(meta, .content = "{\"access_token\":\"X\",\"token_type\":\"X\",\"expires_in\":3600}"); + hrnServerScriptClose(meta); + + // Meta service no longer needed + hrnServerScriptEnd(meta); + testRequestP(service, HTTP_VERB_GET, .object = "file0.txt", .query = "alt=media"); testResponseP(service);