@@ -276,6 +293,8 @@
+
+
GCS support for repository storage.
@@ -9575,6 +9594,11 @@
farrellit
+
+ Daniel Farina
+ fdr
+
+
Daniel Westermann
danielwestermann
diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml
index 7ec021e4b..0ba8005e0 100644
--- a/doc/xml/user-guide.xml
+++ b/doc/xml/user-guide.xml
@@ -669,6 +669,8 @@
4
+
+ When running in GCE set repo{[gcs-setup-repo-id]}-gcs-key-type=auto to automatically authenticate using the instance service account.
diff --git a/src/build/config/config.yaml b/src/build/config/config.yaml
index 677929fb5..8e7038ecf 100644
--- a/src/build/config/config.yaml
+++ b/src/build/config/config.yaml
@@ -1536,7 +1536,11 @@ option:
group: repo
secure: true
command: repo-type
- depend: repo-gcs-bucket
+ depend:
+ option: repo-gcs-key-type
+ list:
+ - service
+ - token
repo-gcs-key-type:
section: global
@@ -1544,6 +1548,7 @@ option:
group: repo
default: service
allow-list:
+ - auto
- service
- token
command: repo-type
diff --git a/src/command/help/help.auto.c b/src/command/help/help.auto.c
index ddb65f66e..fbf80674b 100644
--- a/src/command/help/help.auto.c
+++ b/src/command/help/help.auto.c
@@ -2907,10 +2907,13 @@ static const unsigned char helpDataPack[] =
pckTypeStr << 4 | 0x08, 0x18, // Summary
0x47, 0x43, 0x53, 0x20, 0x72, 0x65, 0x70, 0x6F, 0x73, 0x69, 0x74, 0x6F, 0x72, 0x79, 0x20, 0x6B, 0x65, 0x79, 0x20, 0x74,
0x79, 0x70, 0x65, 0x2E,
- pckTypeStr << 4 | 0x08, 0x95, 0x01, // Description
+ pckTypeStr << 4 | 0x08, 0xCC, 0x01, // Description
0x54, 0x68, 0x65, 0x20, 0x66, 0x6F, 0x6C, 0x6C, 0x6F, 0x77, 0x69, 0x6E, 0x67, 0x20, 0x74, 0x79, 0x70, 0x65, 0x73, 0x20,
0x61, 0x72, 0x65, 0x20, 0x73, 0x75, 0x70, 0x70, 0x6F, 0x72, 0x74, 0x65, 0x64, 0x20, 0x66, 0x6F, 0x72, 0x20, 0x61, 0x75,
0x74, 0x68, 0x6F, 0x72, 0x69, 0x7A, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x3A, 0x0A, 0x0A,
+ 0x2A, 0x20, 0x61, 0x75, 0x74, 0x6F, 0x20, 0x2D, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6F, 0x72, 0x69, 0x7A, 0x65, 0x20, 0x75,
+ 0x73, 0x69, 0x6E, 0x67, 0x20, 0x74, 0x68, 0x65, 0x20, 0x69, 0x6E, 0x73, 0x74, 0x61, 0x6E, 0x63, 0x65, 0x20, 0x73, 0x65,
+ 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x61, 0x63, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x2E, 0x0A,
0x2A, 0x20, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x2D, 0x20, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20,
0x61, 0x63, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x20, 0x66, 0x72, 0x6F, 0x6D, 0x20, 0x6C, 0x6F, 0x63, 0x61, 0x6C, 0x6C, 0x79,
0x20, 0x73, 0x74, 0x6F, 0x72, 0x65, 0x64, 0x20, 0x6B, 0x65, 0x79, 0x2E, 0x0A,
diff --git a/src/config/config.auto.h b/src/config/config.auto.h
index b05370f65..ddd120329 100644
--- a/src/config/config.auto.h
+++ b/src/config/config.auto.h
@@ -187,6 +187,7 @@ Option value constants
#define CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC_Z "aes-256-cbc"
#define CFGOPTVAL_REPO_CIPHER_TYPE_NONE_Z "none"
+#define CFGOPTVAL_REPO_GCS_KEY_TYPE_AUTO_Z "auto"
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_SERVICE_Z "service"
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_TOKEN_Z "token"
diff --git a/src/config/parse.auto.c b/src/config/parse.auto.c
index 250a8d641..278b58b5a 100644
--- a/src/config/parse.auto.c
+++ b/src/config/parse.auto.c
@@ -3795,7 +3795,12 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
PARSE_RULE_OPTION_OPTIONAL_LIST
(
- PARSE_RULE_OPTION_OPTIONAL_DEPEND(cfgOptRepoGcsKeyType),
+ PARSE_RULE_OPTION_OPTIONAL_DEPEND_LIST
+ (
+ cfgOptRepoGcsKeyType,
+ "service",
+ "token"
+ ),
),
),
@@ -3866,6 +3871,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
(
PARSE_RULE_OPTION_OPTIONAL_ALLOW_LIST
(
+ "auto",
"service",
"token"
),
diff --git a/src/storage/gcs/storage.c b/src/storage/gcs/storage.c
index 55a848641..2e8e08ce8 100644
--- a/src/storage/gcs/storage.c
+++ b/src/storage/gcs/storage.c
@@ -31,6 +31,8 @@ GCS Storage
HTTP headers
***********************************************************************************************************************************/
STRING_EXTERN(GCS_HEADER_UPLOAD_ID_STR, GCS_HEADER_UPLOAD_ID);
+STRING_STATIC(GCS_HEADER_METADATA_FLAVOR_STR, "metadata-flavor");
+STRING_STATIC(GCS_HEADER_GOOGLE_STR, "Google");
/***********************************************************************************************************************************
Query tokens
@@ -269,6 +271,41 @@ storageGcsAuthService(StorageGcs *this, time_t timeBegin)
FUNCTION_TEST_RETURN(result);
}
+/***********************************************************************************************************************************
+Get authentication token automatically for instances running in GCE.
+
+Based on the documentation at https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#applications
+***********************************************************************************************************************************/
+static StorageGcsAuthTokenResult
+storageGcsAuthAuto(StorageGcs *this, time_t timeBegin)
+{
+ FUNCTION_TEST_BEGIN();
+ FUNCTION_TEST_PARAM(STORAGE_GCS, this);
+ FUNCTION_TEST_PARAM(TIME, timeBegin);
+ FUNCTION_TEST_END();
+
+ ASSERT(this != NULL);
+ ASSERT(timeBegin > 0);
+
+ StorageGcsAuthTokenResult result = {0};
+
+ MEM_CONTEXT_TEMP_BEGIN()
+ {
+ HttpHeader *header = httpHeaderNew(NULL);
+ httpHeaderAdd(header, HTTP_HEADER_HOST_STR, httpUrlHost(this->authUrl));
+ httpHeaderAdd(header, GCS_HEADER_METADATA_FLAVOR_STR, GCS_HEADER_GOOGLE_STR);
+ httpHeaderAdd(header, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
+
+ HttpRequest *request = httpRequestNewP(
+ this->authClient, HTTP_VERB_GET_STR, httpUrlPath(this->authUrl), NULL, .header = header);
+
+ result = storageGcsAuthToken(request, timeBegin);
+ }
+ MEM_CONTEXT_TEMP_END();
+
+ FUNCTION_TEST_RETURN(result);
+}
+
/***********************************************************************************************************************************
Generate authorization header and add it to the supplied header list
***********************************************************************************************************************************/
@@ -286,37 +323,34 @@ storageGcsAuth(StorageGcs *this, HttpHeader *httpHeader)
MEM_CONTEXT_TEMP_BEGIN()
{
- // Process authentication type
- switch (this->keyType)
+ // Get the token if it was not supplied by the user
+ if (this->keyType != storageGcsKeyTypeToken)
{
- // Service key authentication requests a token then drops through to normal token authentication
- case storageGcsKeyTypeService:
+ ASSERT(this->keyType == storageGcsKeyTypeAuto || this->keyType == storageGcsKeyTypeService);
+
+ time_t timeBegin = time(NULL);
+
+ // If the current token has expired then request a new one
+ if (timeBegin >= this->tokenTimeExpire)
{
- time_t timeBegin = time(NULL);
+ StorageGcsAuthTokenResult tokenResult = this->keyType == storageGcsKeyTypeAuto ?
+ storageGcsAuthAuto(this, timeBegin) : storageGcsAuthService(this, timeBegin);
- // If the current token has expired then request a new one
- if (timeBegin >= this->tokenTimeExpire)
+ MEM_CONTEXT_BEGIN(this->memContext)
{
- StorageGcsAuthTokenResult tokenResult = storageGcsAuthService(this, timeBegin);
+ strFree(this->token);
+ this->token = strNewFmt("%s %s", strZ(tokenResult.tokenType), strZ(tokenResult.token));
- MEM_CONTEXT_BEGIN(this->memContext)
- {
- strFree(this->token);
- this->token = strNewFmt("%s %s", strZ(tokenResult.tokenType), strZ(tokenResult.token));
-
- // Subtract http client timeout * 2 so the token does not expire in the middle of http retries
- this->tokenTimeExpire =
- tokenResult.timeExpire - ((time_t)(httpClientTimeout(this->httpClient) / MSEC_PER_SEC * 2));
- }
- MEM_CONTEXT_END();
+ // Subtract http client timeout * 2 so the token does not expire in the middle of http retries
+ this->tokenTimeExpire =
+ tokenResult.timeExpire - ((time_t)(httpClientTimeout(this->httpClient) / MSEC_PER_SEC * 2));
}
+ MEM_CONTEXT_END();
}
-
- // Token authentication
- case storageGcsKeyTypeToken:
- httpHeaderPut(httpHeader, HTTP_HEADER_AUTHORIZATION_STR, this->token);
- break;
}
+
+ // Add authorization header
+ httpHeaderPut(httpHeader, HTTP_HEADER_AUTHORIZATION_STR, this->token);
}
MEM_CONTEXT_TEMP_END();
@@ -895,7 +929,7 @@ storageGcsNew(
ASSERT(path != NULL);
ASSERT(bucket != NULL);
- ASSERT(key != NULL);
+ ASSERT(keyType == storageGcsKeyTypeAuto || key != NULL);
ASSERT(chunkSize != 0);
Storage *this = NULL;
@@ -917,6 +951,18 @@ storageGcsNew(
// Handle auth key types
switch (keyType)
{
+ // Auto authentication for GCE instances
+ case storageGcsKeyTypeAuto:
+ {
+ driver->authUrl = httpUrlNewParseP(
+ STRDEF("metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"),
+ .type = httpProtocolTypeHttp);
+ driver->authClient = httpClientNew(
+ sckClientNew(httpUrlHost(driver->authUrl), httpUrlPort(driver->authUrl), timeout), timeout);
+
+ break;
+ }
+
// Read data from file for service keys
case storageGcsKeyTypeService:
{
diff --git a/src/storage/gcs/storage.h b/src/storage/gcs/storage.h
index df23410d1..7d5bdd5a2 100644
--- a/src/storage/gcs/storage.h
+++ b/src/storage/gcs/storage.h
@@ -16,6 +16,7 @@ Key type
***********************************************************************************************************************************/
typedef enum
{
+ storageGcsKeyTypeAuto = STRID5("auto", 0x7d2a10),
storageGcsKeyTypeService = STRID5("service", 0x1469b48b30),
storageGcsKeyTypeToken = STRID5("token", 0xe2adf40),
} StorageGcsKeyType;
diff --git a/src/storage/helper.c b/src/storage/helper.c
index e22aee849..1aec9c38e 100644
--- a/src/storage/helper.c
+++ b/src/storage/helper.c
@@ -381,8 +381,9 @@ storageRepoGet(unsigned int repoIdx, bool write)
result = storageGcsNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
- (StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsKey, repoIdx),
- STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
+ (StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx),
+ cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx), STORAGE_GCS_CHUNKSIZE_DEFAULT,
+ cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
break;
diff --git a/test/src/module/storage/gcsTest.c b/test/src/module/storage/gcsTest.c
index 39134aafc..96c5da790 100644
--- a/test/src/module/storage/gcsTest.c
+++ b/test/src/module/storage/gcsTest.c
@@ -190,6 +190,7 @@ testRun(void)
const String *const testHost = hrnServerHost();
const unsigned int testPort = hrnServerPort(0);
const unsigned int testPortAuth = hrnServerPort(1);
+ const unsigned int testPortMeta = hrnServerPort(2);
// *****************************************************************************************************************************
if (testBegin("storageRepoGet()"))
@@ -298,12 +299,24 @@ testRun(void)
}
HARNESS_FORK_CHILD_END();
+ HARNESS_FORK_CHILD_BEGIN(0, true)
+ {
+ TEST_RESULT_VOID(
+ hrnServerRunP(
+ ioFdReadNew(strNew("meta server read"), HARNESS_FORK_CHILD_READ(), 10000), hrnServerProtocolSocket,
+ .port = testPortMeta),
+ "meta server run");
+ }
+ HARNESS_FORK_CHILD_END();
+
HARNESS_FORK_PARENT_BEGIN()
{
IoWrite *service = hrnServerScriptBegin(
ioFdWriteNew(strNew("gcs client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000));
IoWrite *auth = hrnServerScriptBegin(
ioFdWriteNew(strNew("auth client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(1), 2000));
+ IoWrite *meta = hrnServerScriptBegin(
+ ioFdWriteNew(strNew("meta client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(2), 2000));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("test service auth");
@@ -317,13 +330,11 @@ testRun(void)
hrnCfgArgRawBool(argList, cfgOptRepoStorageVerifyTls, testContainer());
hrnCfgEnvRawZ(cfgOptRepoGcsKey, TEST_KEY_FILE);
harnessCfgLoad(cfgCmdArchivePush, argList);
+ hrnCfgEnvRemoveRaw(cfgOptRepoGcsKey);
Storage *storage = NULL;
TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage");
- // Tests need the chunk size to be 16
- ((StorageGcs *)storageDriver(storage))->chunkSize = 16;
-
// Generate the auth request. The JWT part will need to be ? since it can vary in content and size.
const char *const preamble = "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=";
const String *const jwt = storageGcsAuthJwt(((StorageGcs *)storageDriver(storage)), time(NULL));
@@ -407,9 +418,47 @@ testRun(void)
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(storage, strNew("file.txt")))), "this is a sample file", "get file");
+ // -----------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("switch to auto auth");
+
+ hrnServerScriptClose(service);
+
+ StringList *argListAuto = strLstDup(argList);
+ hrnCfgArgRawStrId(argListAuto, cfgOptRepoGcsKeyType, storageGcsKeyTypeAuto);
+ harnessCfgLoad(cfgCmdArchivePush, argListAuto);
+
+ TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage");
+
+ // Replace the default authClient with one that points locally. The default host and url will still be used so they
+ // can be verified when testing auth.
+ ((StorageGcs *)storageDriver(storage))->authClient = httpClientNew(
+ sckClientNew(hrnServerHost(), testPortMeta, 2000), 2000);
+
+ // Tests need the chunk size to be 16
+ ((StorageGcs *)storageDriver(storage))->chunkSize = 16;
+
+ hrnServerScriptAccept(service);
+
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get zero-length file");
+ // Get token automatically from metadata
+ hrnServerScriptAccept(meta);
+ hrnServerScriptExpectZ(
+ meta,
+ "GET /computeMetadata/v1/instance/service-accounts/default/token HTTP/1.1\r\n"
+ "user-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n"
+ "content-length:0\r\n"
+ "host:metadata.google.internal\r\n"
+ "metadata-flavor:Google\r\n"
+ "\r\n");
+
+ testResponseP(meta, .content = "{\"access_token\":\"X\",\"token_type\":\"X\",\"expires_in\":3600}");
+ hrnServerScriptClose(meta);
+
+ // Meta service no longer needed
+ hrnServerScriptEnd(meta);
+
testRequestP(service, HTTP_VERB_GET, .object = "file0.txt", .query = "alt=media");
testResponseP(service);