Add WebIdentity authentication for AWS S3.

This allows credentials to be automatically acquired in an EKS environment.

doc/xml/release.xml

@@ -77,6 +77,21 @@
                         <p>Increase max index allowed for <id>pg</id>/<id>repo</id> options to 256.</p>
                     </release-item>
 
+                    <release-item>
+                        <github-issue id="1203"/>
+                        <github-pull-request id="1527"/>
+
+                        <release-item-contributor-list>
+                            <release-item-contributor id="david.steele"/>
+                            <release-item-reviewer id="james.callahan"/>
+                            <!-- Actually tester, but we don't have a tag for that yet -->
+                            <release-item-reviewer id="benjamin.blattberg"/>
+                            <release-item-reviewer id="andrew.lecuyer"/>
+                        </release-item-contributor-list>
+
+                        <p>Add <id>WebIdentity</id> authentication for <proper>AWS S3</proper>.</p>
+                    </release-item>
+
                     <release-item>
                         <github-issue id="1484"/>
                         <github-pull-request id="1508"/>
@@ -10142,6 +10157,11 @@
             <contributor-id type="github">bastianwegge</contributor-id>
         </contributor>
 
+        <contributor id="benjamin.blattberg">
+            <contributor-name-display>Benjamin Blattberg</contributor-name-display>
+            <contributor-id type="github">benjaminjb</contributor-id>
+        </contributor>
+
         <contributor id="benoit.lobréau">
             <contributor-name-display>blogh</contributor-name-display>
             <contributor-id type="github">blogh</contributor-id>
@@ -10424,6 +10444,11 @@
             <contributor-id type="github">openfirmware</contributor-id>
         </contributor>
 
+        <contributor id="james.callahan">
+            <contributor-name-display>James Callahan</contributor-name-display>
+            <contributor-id type="github">james-callahan</contributor-id>
+        </contributor>
+
         <contributor id="james.chanco.jr">
             <contributor-name-display>James Chanco Jr</contributor-name-display>
             <contributor-id type="github">jameschancojr</contributor-id>

src/build/config/config.yaml

@@ -2092,6 +2092,7 @@ option:
     allow-list:
       - shared
       - auto
+      - web-id
 
   repo-s3-region:
     inherit: repo-s3-bucket

src/build/help/help.xml

@@ -831,6 +831,7 @@
                             <list>
                                 <list-item><id>shared</id> - Shared keys</list-item>
                                 <list-item><id>auto</id> - Automatically retrieve temporary credentials</list-item>
+                                <list-item><id>web-id</id> - Automatically retrieve web identity credentials</list-item>
                             </list>
                         </text>
 

src/config/config.auto.h

@@ -249,6 +249,8 @@ Option value constants
 #define CFGOPTVAL_REPO_S3_KEY_TYPE_AUTO_Z                           "auto"
 #define CFGOPTVAL_REPO_S3_KEY_TYPE_SHARED                           STRID5("shared", 0x85905130)
 #define CFGOPTVAL_REPO_S3_KEY_TYPE_SHARED_Z                         "shared"
+#define CFGOPTVAL_REPO_S3_KEY_TYPE_WEB_ID                           STRID5("web-id", 0x89d88b70)
+#define CFGOPTVAL_REPO_S3_KEY_TYPE_WEB_ID_Z                         "web-id"
 
 #define CFGOPTVAL_REPO_S3_URI_STYLE_HOST                            STRID5("host", 0xa4de80)
 #define CFGOPTVAL_REPO_S3_URI_STYLE_HOST_Z                          "host"

src/config/parse.auto.c

@@ -156,6 +156,7 @@ static const StringId parseRuleValueStrId[] =
     STRID5("token", 0xe2adf40),
     STRID5("trace", 0x5186540),
     STRID5("warn", 0x748370),
+    STRID5("web-id", 0x89d88b70),
     STRID5("xid", 0x11380),
     STRID5("zst", 0x527a0),
 };
@@ -208,6 +209,7 @@ typedef enum
     parseRuleValStrIdToken,
     parseRuleValStrIdTrace,
     parseRuleValStrIdWarn,
+    parseRuleValStrIdWebId,
     parseRuleValStrIdXid,
     parseRuleValStrIdZst,
 } ParseRuleValueStrId;
@@ -6723,6 +6725,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
                 (
                     PARSE_RULE_VAL_STRID(parseRuleValStrIdShared),
                     PARSE_RULE_VAL_STRID(parseRuleValStrIdAuto),
+                    PARSE_RULE_VAL_STRID(parseRuleValStrIdWebId),
                 ),
 
                 PARSE_RULE_OPTIONAL_DEFAULT

src/storage/s3/helper.c

@@ -3,10 +3,13 @@ S3 Storage Helper
 ***********************************************************************************************************************************/
 #include "build.auto.h"
 
+#include <stdlib.h>
+
 #include "common/debug.h"
 #include "common/io/io.h"
 #include "common/log.h"
 #include "config/config.h"
+#include "storage/posix/storage.h"
 #include "storage/s3/helper.h"
 
 /**********************************************************************************************************************************/
@@ -32,14 +35,41 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
     if (cfgOptionIdxSource(cfgOptRepoStoragePort, repoIdx) != cfgSourceDefault)
         port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
 
+    // Get role and token
+    const StorageS3KeyType keyType = (StorageS3KeyType)cfgOptionIdxStrId(cfgOptRepoS3KeyType, repoIdx);
+    const String *role = cfgOptionIdxStrNull(cfgOptRepoS3Role, repoIdx);
+    const String *webIdToken = NULL;
+
+    // If web identity authentication then load the role and token from environment variables documented here:
+    // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html
+    if (keyType == storageS3KeyTypeWebId)
+    {
+        #define S3_ENV_AWS_ROLE_ARN                             "AWS_ROLE_ARN"
+        #define S3_ENV_AWS_WEB_IDENTITY_TOKEN_FILE              "AWS_WEB_IDENTITY_TOKEN_FILE"
+
+        const char *const roleZ = getenv(S3_ENV_AWS_ROLE_ARN);
+        const char *const webIdTokenFileZ = getenv(S3_ENV_AWS_WEB_IDENTITY_TOKEN_FILE);
+
+        if (roleZ == NULL || webIdTokenFileZ == NULL)
+        {
+            THROW_FMT(
+                OptionInvalidError,
+                "option '%s' is '" CFGOPTVAL_REPO_S3_KEY_TYPE_WEB_ID_Z "' but '" S3_ENV_AWS_ROLE_ARN "' and '"
+                    S3_ENV_AWS_WEB_IDENTITY_TOKEN_FILE "' are not set",
+                cfgOptionIdxName(cfgOptRepoS3KeyType, repoIdx));
+        }
+
+        role = strNewZ(roleZ);
+        webIdToken = strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), STR(webIdTokenFileZ))));
+    }
+
     Storage *const result = storageS3New(
-        cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx),
-        endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx),
-        (StorageS3KeyType)cfgOptionIdxStrId(cfgOptRepoS3KeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx),
-        cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx),
-        cfgOptionIdxStrNull(cfgOptRepoS3Role, repoIdx), STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(),
-        cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
-        cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
+        cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback,
+        cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx),
+        cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx),
+        cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), role,
+        webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
+        cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
 
     FUNCTION_LOG_RETURN(STORAGE, result);
 }

src/storage/s3/storage.c

@@ -99,6 +99,7 @@ struct StorageS3
     HttpClient *credHttpClient;                                     // HTTP client to service credential requests
     const String *credHost;                                         // Credentials host
     const String *credRole;                                         // Role to use for credential requests
+    const String *webIdToken;                                       // Token to use for credential requests
     time_t credExpirationTime;                                      // Time the temporary credentials expire
 
     // Current signing key and date it is valid for
@@ -354,6 +355,61 @@ storageS3AuthAuto(StorageS3 *const this, const HttpHeader *const header)
     FUNCTION_LOG_RETURN_VOID();
 }
 
+/***********************************************************************************************************************************
+Automatically get credentials for an associated web identity service account
+
+Documentation is found at: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
+***********************************************************************************************************************************/
+STRING_STATIC(S3_STS_HOST_STR,                                      "sts.amazonaws.com");
+#define S3_STS_PORT                                                 443
+
+static void
+storageS3AuthWebId(StorageS3 *const this, const HttpHeader *const header)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(STORAGE_S3, this);
+        FUNCTION_LOG_PARAM(HTTP_HEADER, header);
+    FUNCTION_LOG_END();
+
+    // Get credentials
+    HttpQuery *const query = httpQueryNewP();
+    httpQueryAdd(query, STRDEF("Action"), STRDEF("AssumeRoleWithWebIdentity"));
+    httpQueryAdd(query, STRDEF("RoleArn"), this->credRole);
+    httpQueryAdd(query, STRDEF("RoleSessionName"), STRDEF(PROJECT_NAME));
+    httpQueryAdd(query, STRDEF("Version"), STRDEF("2011-06-15"));
+    httpQueryAdd(query, STRDEF("WebIdentityToken"), this->webIdToken);
+
+    HttpRequest *const request = httpRequestNewP(
+        this->credHttpClient, HTTP_VERB_GET_STR, FSLASH_STR, .header = header, .query = query);
+    HttpResponse *const response = httpRequestResponse(request, true);
+
+    CHECK(httpResponseCode(response) != HTTP_RESPONSE_CODE_NOT_FOUND);
+
+    // Copy credentials
+    const XmlNode *const xmlCred =
+        xmlNodeChild(
+            xmlNodeChild(
+                xmlDocumentRoot(xmlDocumentNewBuf(httpResponseContent(response))), STRDEF("AssumeRoleWithWebIdentityResult"), true),
+            STRDEF("Credentials"), true);
+
+    const XmlNode *const accessKeyNode = xmlNodeChild(xmlCred, STRDEF("AccessKeyId"), true);
+    const XmlNode *const secretAccessKeyNode = xmlNodeChild(xmlCred, STRDEF("SecretAccessKey"), true);
+    const XmlNode *const securityTokenNode = xmlNodeChild(xmlCred, STRDEF("SessionToken"), true);
+
+    MEM_CONTEXT_BEGIN(THIS_MEM_CONTEXT())
+    {
+        this->accessKey = xmlNodeContent(accessKeyNode);
+        this->secretAccessKey = xmlNodeContent(secretAccessKeyNode);
+        this->securityToken = xmlNodeContent(securityTokenNode);
+    }
+    MEM_CONTEXT_END();
+
+    // Update expiration time
+    this->credExpirationTime = storageS3CvtTime(xmlNodeContent(xmlNodeChild(xmlCred, STRDEF("Expiration"), true)));
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
 /***********************************************************************************************************************************
 Process S3 request
 ***********************************************************************************************************************************/
@@ -409,7 +465,22 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S
             httpHeaderAdd(credHeader, HTTP_HEADER_HOST_STR, this->credHost);
 
             // Get credentials
-            storageS3AuthAuto(this, credHeader);
+            switch (this->keyType)
+            {
+                // Auto authentication
+                case storageS3KeyTypeAuto:
+                    storageS3AuthAuto(this, credHeader);
+                    break;
+
+                // Web identity authentication
+                default:
+                {
+                    ASSERT(this->keyType == storageS3KeyTypeWebId);
+
+                    storageS3AuthWebId(this, credHeader);
+                    break;
+                }
+            }
 
             // Reset the signing key date so the signing key gets regenerated
             this->signingKeyDate = YYYYMMDD_STR;
@@ -946,8 +1017,9 @@ Storage *
 storageS3New(
     const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
     const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
-    const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host,
-    unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
+    const String *secretAccessKey, const String *securityToken, const String *credRole, const String *const webIdToken,
+    size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile,
+    const String *caPath)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(STRING, path);
@@ -962,6 +1034,7 @@ storageS3New(
         FUNCTION_TEST_PARAM(STRING, secretAccessKey);
         FUNCTION_TEST_PARAM(STRING, securityToken);
         FUNCTION_TEST_PARAM(STRING, credRole);
+        FUNCTION_TEST_PARAM(STRING, webIdToken);
         FUNCTION_LOG_PARAM(SIZE, partSize);
         FUNCTION_LOG_PARAM(STRING, host);
         FUNCTION_LOG_PARAM(UINT, port);
@@ -1025,6 +1098,26 @@ storageS3New(
                 break;
             }
 
+            // Create the HTTP client used to retrieve web identity security credentials
+            case storageS3KeyTypeWebId:
+            {
+                ASSERT(accessKey == NULL && secretAccessKey == NULL && securityToken == NULL);
+                ASSERT(credRole != NULL);
+                ASSERT(webIdToken != NULL);
+
+                driver->credRole = strDup(credRole);
+                driver->webIdToken = strDup(webIdToken);
+                driver->credHost = S3_STS_HOST_STR;
+                driver->credExpirationTime = time(NULL);
+                driver->credHttpClient = httpClientNew(
+                    tlsClientNew(
+                        sckClientNew(driver->credHost, S3_STS_PORT, timeout, timeout), driver->credHost, timeout, true, caFile,
+                        caPath, NULL, NULL, NULL),
+                    timeout);
+
+                break;
+            }
+
             // Set shared key credentials
             default:
             {

src/storage/s3/storage.h

@@ -18,6 +18,7 @@ typedef enum
 {
     storageS3KeyTypeShared = STRID5("shared", 0x85905130),
     storageS3KeyTypeAuto = STRID5("auto", 0x7d2a10),
+    storageS3KeyTypeWebId = STRID5("web-id", 0x89d88b70),
 } StorageS3KeyType;
 
 /***********************************************************************************************************************************
@@ -40,7 +41,7 @@ Constructors
 Storage *storageS3New(
     const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
     const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
-    const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host,
-    unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
+    const String *secretAccessKey, const String *securityToken, const String *credRole, const String *webIdToken, size_t partSize,
+    const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
 
 #endif

test/src/module/storage/s3Test.c

@@ -645,9 +645,6 @@ testRun(void)
                 // Check that the signing key changed
                 TEST_RESULT_BOOL(bufEq(driver->signingKey, oldSigningKey), false, "signing key changed");
 
-                // Auth service no longer needed
-                hrnServerScriptEnd(auth);
-
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("write zero-length file");
 
@@ -798,11 +795,78 @@ testRun(void)
 
                 TEST_RESULT_BOOL(storageInfoP(s3, NULL, .ignoreMissing = true).exists, false, "info for /");
 
+                // -----------------------------------------------------------------------------------------------------------------
+                TEST_TITLE("switch to service credentials");
+
+                hrnServerScriptClose(service);
+
+                #define TEST_SERVICE_ROLE                           "arn:aws:iam::123456789012:role/TestRole"
+                #define TEST_SERVICE_TOKEN                          "TOKEN"
+                #define TEST_SERVICE_URI                                                                                           \
+                    "/?Action=AssumeRoleWithWebIdentity&RoleArn=arn%3Aaws%3Aiam%3A%3A123456789012%3Arole%2FTestRole"               \
+                        "&RoleSessionName=pgBackRest&Version=2011-06-15&WebIdentityToken=" TEST_SERVICE_TOKEN
+                #define TEST_SERVICE_RESPONSE                                                                                      \
+                    "<AssumeRoleWithWebIdentityResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">\n"                    \
+                    "  <AssumeRoleWithWebIdentityResult>\n"                                                                        \
+                    "    <Credentials>\n"                                                                                          \
+                    "      <SessionToken>zz</SessionToken>\n"                                                                      \
+                    "      <SecretAccessKey>yy</SecretAccessKey>\n"                                                                \
+                    "      <Expiration>%s</Expiration>\n"                                                                          \
+                    "      <AccessKeyId>xx</AccessKeyId>\n"                                                                        \
+                    "    </Credentials>\n"                                                                                         \
+                    "  </AssumeRoleWithWebIdentityResult>\n"                                                                       \
+                    "</AssumeRoleWithWebIdentityResponse>"
+
+                HRN_STORAGE_PUT_Z(storagePosixNewP(TEST_PATH_STR, .write = true), "web-id-token", TEST_SERVICE_TOKEN);
+
+                argList = strLstDup(commonArgList);
+                hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), port);
+                hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeWebId);
+                HRN_CFG_LOAD(cfgCmdArchivePush, argList);
+
+                TEST_ERROR(
+                    storageRepoGet(0, true), OptionInvalidError,
+                    "option 'repo1-s3-key-type' is 'web-id' but 'AWS_ROLE_ARN' and 'AWS_WEB_IDENTITY_TOKEN_FILE' are not set");
+
+                setenv("AWS_ROLE_ARN", TEST_SERVICE_ROLE, true);
+
+                TEST_ERROR(
+                    storageRepoGet(0, true), OptionInvalidError,
+                    "option 'repo1-s3-key-type' is 'web-id' but 'AWS_ROLE_ARN' and 'AWS_WEB_IDENTITY_TOKEN_FILE' are not set");
+
+                setenv("AWS_WEB_IDENTITY_TOKEN_FILE", TEST_PATH "/web-id-token", true);
+
+                s3 = storageRepoGet(0, true);
+                driver = (StorageS3 *)storageDriver(s3);
+
+                TEST_RESULT_STR_Z(driver->credRole, TEST_SERVICE_ROLE, "check role");
+                TEST_RESULT_STR_Z(driver->webIdToken, TEST_SERVICE_TOKEN, "check token");
+
+                // Set partSize to a small value for testing
+                driver->partSize = 16;
+
+                // Testing requires the auth http client to be redirected
+                driver->credHost = hrnServerHost();
+                driver->credHttpClient = httpClientNew(sckClientNew(host, authPort, 5000, 5000), 5000);
+
+                hrnServerScriptAccept(service);
+
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("info for missing file");
 
+                // Get service credentials
+                hrnServerScriptAccept(auth);
+
+                testRequestP(auth, NULL, HTTP_VERB_GET, TEST_SERVICE_URI);
+                testResponseP(
+                    auth,
+                    .content = strZ(
+                        strNewFmt(TEST_SERVICE_RESPONSE, strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC - 1))))));
+
+                hrnServerScriptClose(auth);
+
                 // File missing
-                testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS");
+                testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS", .accessKey = "xx", .securityToken = "zz");
                 testResponseP(service, .code = 404);
 
                 TEST_RESULT_BOOL(storageInfoP(s3, STRDEF("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist");
@@ -810,6 +874,17 @@ testRun(void)
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("info for file");
 
+                // Get service credentials
+                hrnServerScriptAccept(auth);
+
+                testRequestP(auth, NULL, HTTP_VERB_GET, TEST_SERVICE_URI);
+                testResponseP(
+                    auth,
+                    .content = strZ(
+                        strNewFmt(TEST_SERVICE_RESPONSE, strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC * 2))))));
+
+                hrnServerScriptClose(auth);
+
                 testRequestP(service, s3, HTTP_VERB_HEAD, "/subdir/file1.txt");
                 testResponseP(service, .header = "content-length:9999\r\nLast-Modified: Wed, 21 Oct 2015 07:28:00 GMT");
 
@@ -820,6 +895,9 @@ testRun(void)
                 TEST_RESULT_UINT(info.size, 9999, "check exists");
                 TEST_RESULT_INT(info.timeModified, 1445412480, "check time");
 
+                // Auth service no longer needed
+                hrnServerScriptEnd(auth);
+
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("info check existence only");