From 3879bc69b888daa04d2ca98a2d1219cf22519ddc Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 22 Oct 2021 18:31:55 -0400 Subject: [PATCH] Add WebIdentity authentication for AWS S3. This allows credentials to be automatically acquired in an EKS environment. --- doc/xml/release.xml | 25 ++++++++ src/build/config/config.yaml | 1 + src/build/help/help.xml | 1 + src/config/config.auto.h | 2 + src/config/parse.auto.c | 3 + src/storage/s3/helper.c | 44 +++++++++++--- src/storage/s3/storage.c | 99 +++++++++++++++++++++++++++++++- src/storage/s3/storage.h | 5 +- test/src/module/storage/s3Test.c | 86 +++++++++++++++++++++++++-- 9 files changed, 250 insertions(+), 16 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 43826ae91..7fc644cd5 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -77,6 +77,21 @@

Increase max index allowed for pg/repo options to 256.

+ + + + + + + + + + + + +

Add WebIdentity authentication for AWS S3.

+ + @@ -10142,6 +10157,11 @@ bastianwegge + + Benjamin Blattberg + benjaminjb + + blogh blogh @@ -10424,6 +10444,11 @@ openfirmware + + James Callahan + james-callahan + + James Chanco Jr jameschancojr diff --git a/src/build/config/config.yaml b/src/build/config/config.yaml index 7de01603c..d3324285b 100644 --- a/src/build/config/config.yaml +++ b/src/build/config/config.yaml @@ -2092,6 +2092,7 @@ option: allow-list: - shared - auto + - web-id repo-s3-region: inherit: repo-s3-bucket diff --git a/src/build/help/help.xml b/src/build/help/help.xml index 7ff867ee5..e69952dea 100644 --- a/src/build/help/help.xml +++ b/src/build/help/help.xml @@ -831,6 +831,7 @@ shared - Shared keys auto - Automatically retrieve temporary credentials + web-id - Automatically retrieve web identity credentials diff --git a/src/config/config.auto.h b/src/config/config.auto.h index c181afd1a..3f40fc7ca 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -249,6 +249,8 @@ Option value constants #define CFGOPTVAL_REPO_S3_KEY_TYPE_AUTO_Z "auto" #define CFGOPTVAL_REPO_S3_KEY_TYPE_SHARED STRID5("shared", 0x85905130) #define CFGOPTVAL_REPO_S3_KEY_TYPE_SHARED_Z "shared" +#define CFGOPTVAL_REPO_S3_KEY_TYPE_WEB_ID STRID5("web-id", 0x89d88b70) +#define CFGOPTVAL_REPO_S3_KEY_TYPE_WEB_ID_Z "web-id" #define CFGOPTVAL_REPO_S3_URI_STYLE_HOST STRID5("host", 0xa4de80) #define CFGOPTVAL_REPO_S3_URI_STYLE_HOST_Z "host" diff --git a/src/config/parse.auto.c b/src/config/parse.auto.c index e9c1985b1..ace8a0ad4 100644 --- a/src/config/parse.auto.c +++ b/src/config/parse.auto.c @@ -156,6 +156,7 @@ static const StringId parseRuleValueStrId[] = STRID5("token", 0xe2adf40), STRID5("trace", 0x5186540), STRID5("warn", 0x748370), + STRID5("web-id", 0x89d88b70), STRID5("xid", 0x11380), STRID5("zst", 0x527a0), }; @@ -208,6 +209,7 @@ typedef enum parseRuleValStrIdToken, parseRuleValStrIdTrace, parseRuleValStrIdWarn, + parseRuleValStrIdWebId, parseRuleValStrIdXid, parseRuleValStrIdZst, } ParseRuleValueStrId; @@ -6723,6 +6725,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = ( PARSE_RULE_VAL_STRID(parseRuleValStrIdShared), PARSE_RULE_VAL_STRID(parseRuleValStrIdAuto), + PARSE_RULE_VAL_STRID(parseRuleValStrIdWebId), ), PARSE_RULE_OPTIONAL_DEFAULT diff --git a/src/storage/s3/helper.c b/src/storage/s3/helper.c index a9b29788e..0e5477cfe 100644 --- a/src/storage/s3/helper.c +++ b/src/storage/s3/helper.c @@ -3,10 +3,13 @@ S3 Storage Helper ***********************************************************************************************************************************/ #include "build.auto.h" +#include + #include "common/debug.h" #include "common/io/io.h" #include "common/log.h" #include "config/config.h" +#include "storage/posix/storage.h" #include "storage/s3/helper.h" /**********************************************************************************************************************************/ @@ -32,14 +35,41 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress if (cfgOptionIdxSource(cfgOptRepoStoragePort, repoIdx) != cfgSourceDefault) port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx); + // Get role and token + const StorageS3KeyType keyType = (StorageS3KeyType)cfgOptionIdxStrId(cfgOptRepoS3KeyType, repoIdx); + const String *role = cfgOptionIdxStrNull(cfgOptRepoS3Role, repoIdx); + const String *webIdToken = NULL; + + // If web identity authentication then load the role and token from environment variables documented here: + // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html + if (keyType == storageS3KeyTypeWebId) + { + #define S3_ENV_AWS_ROLE_ARN "AWS_ROLE_ARN" + #define S3_ENV_AWS_WEB_IDENTITY_TOKEN_FILE "AWS_WEB_IDENTITY_TOKEN_FILE" + + const char *const roleZ = getenv(S3_ENV_AWS_ROLE_ARN); + const char *const webIdTokenFileZ = getenv(S3_ENV_AWS_WEB_IDENTITY_TOKEN_FILE); + + if (roleZ == NULL || webIdTokenFileZ == NULL) + { + THROW_FMT( + OptionInvalidError, + "option '%s' is '" CFGOPTVAL_REPO_S3_KEY_TYPE_WEB_ID_Z "' but '" S3_ENV_AWS_ROLE_ARN "' and '" + S3_ENV_AWS_WEB_IDENTITY_TOKEN_FILE "' are not set", + cfgOptionIdxName(cfgOptRepoS3KeyType, repoIdx)); + } + + role = strNewZ(roleZ); + webIdToken = strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), STR(webIdTokenFileZ)))); + } + Storage *const result = storageS3New( - cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), - endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), - (StorageS3KeyType)cfgOptionIdxStrId(cfgOptRepoS3KeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoS3Role, repoIdx), STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), - cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); + cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, + cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), + cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), + cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), role, + webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), + cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); FUNCTION_LOG_RETURN(STORAGE, result); } diff --git a/src/storage/s3/storage.c b/src/storage/s3/storage.c index fd138327a..1f88666d0 100644 --- a/src/storage/s3/storage.c +++ b/src/storage/s3/storage.c @@ -99,6 +99,7 @@ struct StorageS3 HttpClient *credHttpClient; // HTTP client to service credential requests const String *credHost; // Credentials host const String *credRole; // Role to use for credential requests + const String *webIdToken; // Token to use for credential requests time_t credExpirationTime; // Time the temporary credentials expire // Current signing key and date it is valid for @@ -354,6 +355,61 @@ storageS3AuthAuto(StorageS3 *const this, const HttpHeader *const header) FUNCTION_LOG_RETURN_VOID(); } +/*********************************************************************************************************************************** +Automatically get credentials for an associated web identity service account + +Documentation is found at: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html +***********************************************************************************************************************************/ +STRING_STATIC(S3_STS_HOST_STR, "sts.amazonaws.com"); +#define S3_STS_PORT 443 + +static void +storageS3AuthWebId(StorageS3 *const this, const HttpHeader *const header) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(STORAGE_S3, this); + FUNCTION_LOG_PARAM(HTTP_HEADER, header); + FUNCTION_LOG_END(); + + // Get credentials + HttpQuery *const query = httpQueryNewP(); + httpQueryAdd(query, STRDEF("Action"), STRDEF("AssumeRoleWithWebIdentity")); + httpQueryAdd(query, STRDEF("RoleArn"), this->credRole); + httpQueryAdd(query, STRDEF("RoleSessionName"), STRDEF(PROJECT_NAME)); + httpQueryAdd(query, STRDEF("Version"), STRDEF("2011-06-15")); + httpQueryAdd(query, STRDEF("WebIdentityToken"), this->webIdToken); + + HttpRequest *const request = httpRequestNewP( + this->credHttpClient, HTTP_VERB_GET_STR, FSLASH_STR, .header = header, .query = query); + HttpResponse *const response = httpRequestResponse(request, true); + + CHECK(httpResponseCode(response) != HTTP_RESPONSE_CODE_NOT_FOUND); + + // Copy credentials + const XmlNode *const xmlCred = + xmlNodeChild( + xmlNodeChild( + xmlDocumentRoot(xmlDocumentNewBuf(httpResponseContent(response))), STRDEF("AssumeRoleWithWebIdentityResult"), true), + STRDEF("Credentials"), true); + + const XmlNode *const accessKeyNode = xmlNodeChild(xmlCred, STRDEF("AccessKeyId"), true); + const XmlNode *const secretAccessKeyNode = xmlNodeChild(xmlCred, STRDEF("SecretAccessKey"), true); + const XmlNode *const securityTokenNode = xmlNodeChild(xmlCred, STRDEF("SessionToken"), true); + + MEM_CONTEXT_BEGIN(THIS_MEM_CONTEXT()) + { + this->accessKey = xmlNodeContent(accessKeyNode); + this->secretAccessKey = xmlNodeContent(secretAccessKeyNode); + this->securityToken = xmlNodeContent(securityTokenNode); + } + MEM_CONTEXT_END(); + + // Update expiration time + this->credExpirationTime = storageS3CvtTime(xmlNodeContent(xmlNodeChild(xmlCred, STRDEF("Expiration"), true))); + + FUNCTION_LOG_RETURN_VOID(); +} + /*********************************************************************************************************************************** Process S3 request ***********************************************************************************************************************************/ @@ -409,7 +465,22 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S httpHeaderAdd(credHeader, HTTP_HEADER_HOST_STR, this->credHost); // Get credentials - storageS3AuthAuto(this, credHeader); + switch (this->keyType) + { + // Auto authentication + case storageS3KeyTypeAuto: + storageS3AuthAuto(this, credHeader); + break; + + // Web identity authentication + default: + { + ASSERT(this->keyType == storageS3KeyTypeWebId); + + storageS3AuthWebId(this, credHeader); + break; + } + } // Reset the signing key date so the signing key gets regenerated this->signingKeyDate = YYYYMMDD_STR; @@ -946,8 +1017,9 @@ Storage * storageS3New( const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey, - const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host, - unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath) + const String *secretAccessKey, const String *securityToken, const String *credRole, const String *const webIdToken, + size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, + const String *caPath) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, path); @@ -962,6 +1034,7 @@ storageS3New( FUNCTION_TEST_PARAM(STRING, secretAccessKey); FUNCTION_TEST_PARAM(STRING, securityToken); FUNCTION_TEST_PARAM(STRING, credRole); + FUNCTION_TEST_PARAM(STRING, webIdToken); FUNCTION_LOG_PARAM(SIZE, partSize); FUNCTION_LOG_PARAM(STRING, host); FUNCTION_LOG_PARAM(UINT, port); @@ -1025,6 +1098,26 @@ storageS3New( break; } + // Create the HTTP client used to retrieve web identity security credentials + case storageS3KeyTypeWebId: + { + ASSERT(accessKey == NULL && secretAccessKey == NULL && securityToken == NULL); + ASSERT(credRole != NULL); + ASSERT(webIdToken != NULL); + + driver->credRole = strDup(credRole); + driver->webIdToken = strDup(webIdToken); + driver->credHost = S3_STS_HOST_STR; + driver->credExpirationTime = time(NULL); + driver->credHttpClient = httpClientNew( + tlsClientNew( + sckClientNew(driver->credHost, S3_STS_PORT, timeout, timeout), driver->credHost, timeout, true, caFile, + caPath, NULL, NULL, NULL), + timeout); + + break; + } + // Set shared key credentials default: { diff --git a/src/storage/s3/storage.h b/src/storage/s3/storage.h index 53410f489..6d6b42b9b 100644 --- a/src/storage/s3/storage.h +++ b/src/storage/s3/storage.h @@ -18,6 +18,7 @@ typedef enum { storageS3KeyTypeShared = STRID5("shared", 0x85905130), storageS3KeyTypeAuto = STRID5("auto", 0x7d2a10), + storageS3KeyTypeWebId = STRID5("web-id", 0x89d88b70), } StorageS3KeyType; /*********************************************************************************************************************************** @@ -40,7 +41,7 @@ Constructors Storage *storageS3New( const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey, - const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host, - unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath); + const String *secretAccessKey, const String *securityToken, const String *credRole, const String *webIdToken, size_t partSize, + const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath); #endif diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index 96a7f0276..ebf1911b5 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -645,9 +645,6 @@ testRun(void) // Check that the signing key changed TEST_RESULT_BOOL(bufEq(driver->signingKey, oldSigningKey), false, "signing key changed"); - // Auth service no longer needed - hrnServerScriptEnd(auth); - // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("write zero-length file"); @@ -798,11 +795,78 @@ testRun(void) TEST_RESULT_BOOL(storageInfoP(s3, NULL, .ignoreMissing = true).exists, false, "info for /"); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("switch to service credentials"); + + hrnServerScriptClose(service); + + #define TEST_SERVICE_ROLE "arn:aws:iam::123456789012:role/TestRole" + #define TEST_SERVICE_TOKEN "TOKEN" + #define TEST_SERVICE_URI \ + "/?Action=AssumeRoleWithWebIdentity&RoleArn=arn%3Aaws%3Aiam%3A%3A123456789012%3Arole%2FTestRole" \ + "&RoleSessionName=pgBackRest&Version=2011-06-15&WebIdentityToken=" TEST_SERVICE_TOKEN + #define TEST_SERVICE_RESPONSE \ + "\n" \ + " \n" \ + " \n" \ + " zz\n" \ + " yy\n" \ + " %s\n" \ + " xx\n" \ + " \n" \ + " \n" \ + "" + + HRN_STORAGE_PUT_Z(storagePosixNewP(TEST_PATH_STR, .write = true), "web-id-token", TEST_SERVICE_TOKEN); + + argList = strLstDup(commonArgList); + hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), port); + hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeWebId); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + TEST_ERROR( + storageRepoGet(0, true), OptionInvalidError, + "option 'repo1-s3-key-type' is 'web-id' but 'AWS_ROLE_ARN' and 'AWS_WEB_IDENTITY_TOKEN_FILE' are not set"); + + setenv("AWS_ROLE_ARN", TEST_SERVICE_ROLE, true); + + TEST_ERROR( + storageRepoGet(0, true), OptionInvalidError, + "option 'repo1-s3-key-type' is 'web-id' but 'AWS_ROLE_ARN' and 'AWS_WEB_IDENTITY_TOKEN_FILE' are not set"); + + setenv("AWS_WEB_IDENTITY_TOKEN_FILE", TEST_PATH "/web-id-token", true); + + s3 = storageRepoGet(0, true); + driver = (StorageS3 *)storageDriver(s3); + + TEST_RESULT_STR_Z(driver->credRole, TEST_SERVICE_ROLE, "check role"); + TEST_RESULT_STR_Z(driver->webIdToken, TEST_SERVICE_TOKEN, "check token"); + + // Set partSize to a small value for testing + driver->partSize = 16; + + // Testing requires the auth http client to be redirected + driver->credHost = hrnServerHost(); + driver->credHttpClient = httpClientNew(sckClientNew(host, authPort, 5000, 5000), 5000); + + hrnServerScriptAccept(service); + // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("info for missing file"); + // Get service credentials + hrnServerScriptAccept(auth); + + testRequestP(auth, NULL, HTTP_VERB_GET, TEST_SERVICE_URI); + testResponseP( + auth, + .content = strZ( + strNewFmt(TEST_SERVICE_RESPONSE, strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC - 1)))))); + + hrnServerScriptClose(auth); + // File missing - testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS"); + testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS", .accessKey = "xx", .securityToken = "zz"); testResponseP(service, .code = 404); TEST_RESULT_BOOL(storageInfoP(s3, STRDEF("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist"); @@ -810,6 +874,17 @@ testRun(void) // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("info for file"); + // Get service credentials + hrnServerScriptAccept(auth); + + testRequestP(auth, NULL, HTTP_VERB_GET, TEST_SERVICE_URI); + testResponseP( + auth, + .content = strZ( + strNewFmt(TEST_SERVICE_RESPONSE, strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC * 2)))))); + + hrnServerScriptClose(auth); + testRequestP(service, s3, HTTP_VERB_HEAD, "/subdir/file1.txt"); testResponseP(service, .header = "content-length:9999\r\nLast-Modified: Wed, 21 Oct 2015 07:28:00 GMT"); @@ -820,6 +895,9 @@ testRun(void) TEST_RESULT_UINT(info.size, 9999, "check exists"); TEST_RESULT_INT(info.timeModified, 1445412480, "check time"); + // Auth service no longer needed + hrnServerScriptEnd(auth); + // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("info check existence only");