Keith Fiske
keithf4
diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml
index 2bfe99e8c..310c13a7b 100644
--- a/doc/xml/user-guide.xml
+++ b/doc/xml/user-guide.xml
@@ -2204,7 +2204,9 @@
y
- A role should be created to run and the bucket permissions should be set as restrictively as possible. This sample Amazon S3 policy will restrict all reads and writes to the bucket and repository path.
+ A role should be created to run and the bucket permissions should be set as restrictively as possible. If the role is associated with an instance in AWS then will automatically retrieve temporary credentials when repo1-s3-key-type=auto, which means that keys do not need to be explicitly set in {[backrest-config-demo]}.
+
+ This sample Amazon S3 policy will restrict all reads and writes to the bucket and repository path.
{
diff --git a/src/config/config.auto.c b/src/config/config.auto.c
index a6df642c2..4ba5398b2 100644
--- a/src/config/config.auto.c
+++ b/src/config/config.auto.c
@@ -461,8 +461,10 @@ STRING_EXTERN(CFGOPT_REPO1_S3_ENDPOINT_STR, CFGOPT_REPO1
STRING_EXTERN(CFGOPT_REPO1_S3_HOST_STR, CFGOPT_REPO1_S3_HOST);
STRING_EXTERN(CFGOPT_REPO1_S3_KEY_STR, CFGOPT_REPO1_S3_KEY);
STRING_EXTERN(CFGOPT_REPO1_S3_KEY_SECRET_STR, CFGOPT_REPO1_S3_KEY_SECRET);
+STRING_EXTERN(CFGOPT_REPO1_S3_KEY_TYPE_STR, CFGOPT_REPO1_S3_KEY_TYPE);
STRING_EXTERN(CFGOPT_REPO1_S3_PORT_STR, CFGOPT_REPO1_S3_PORT);
STRING_EXTERN(CFGOPT_REPO1_S3_REGION_STR, CFGOPT_REPO1_S3_REGION);
+STRING_EXTERN(CFGOPT_REPO1_S3_ROLE_STR, CFGOPT_REPO1_S3_ROLE);
STRING_EXTERN(CFGOPT_REPO1_S3_TOKEN_STR, CFGOPT_REPO1_S3_TOKEN);
STRING_EXTERN(CFGOPT_REPO1_S3_URI_STYLE_STR, CFGOPT_REPO1_S3_URI_STYLE);
STRING_EXTERN(CFGOPT_REPO1_S3_VERIFY_TLS_STR, CFGOPT_REPO1_S3_VERIFY_TLS);
@@ -1916,6 +1918,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3KeySecret)
)
+ //------------------------------------------------------------------------------------------------------------------------------
+ CONFIG_OPTION
+ (
+ CONFIG_OPTION_NAME(CFGOPT_REPO1_S3_KEY_TYPE)
+ CONFIG_OPTION_INDEX(0)
+ CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3KeyType)
+ )
+
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
@@ -1932,6 +1942,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3Region)
)
+ //------------------------------------------------------------------------------------------------------------------------------
+ CONFIG_OPTION
+ (
+ CONFIG_OPTION_NAME(CFGOPT_REPO1_S3_ROLE)
+ CONFIG_OPTION_INDEX(0)
+ CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3Role)
+ )
+
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
diff --git a/src/config/config.auto.h b/src/config/config.auto.h
index 29dd6d99a..0e6d8a4c1 100644
--- a/src/config/config.auto.h
+++ b/src/config/config.auto.h
@@ -409,10 +409,14 @@ Option constants
STRING_DECLARE(CFGOPT_REPO1_S3_KEY_STR);
#define CFGOPT_REPO1_S3_KEY_SECRET "repo1-s3-key-secret"
STRING_DECLARE(CFGOPT_REPO1_S3_KEY_SECRET_STR);
+#define CFGOPT_REPO1_S3_KEY_TYPE "repo1-s3-key-type"
+ STRING_DECLARE(CFGOPT_REPO1_S3_KEY_TYPE_STR);
#define CFGOPT_REPO1_S3_PORT "repo1-s3-port"
STRING_DECLARE(CFGOPT_REPO1_S3_PORT_STR);
#define CFGOPT_REPO1_S3_REGION "repo1-s3-region"
STRING_DECLARE(CFGOPT_REPO1_S3_REGION_STR);
+#define CFGOPT_REPO1_S3_ROLE "repo1-s3-role"
+ STRING_DECLARE(CFGOPT_REPO1_S3_ROLE_STR);
#define CFGOPT_REPO1_S3_TOKEN "repo1-s3-token"
STRING_DECLARE(CFGOPT_REPO1_S3_TOKEN_STR);
#define CFGOPT_REPO1_S3_URI_STYLE "repo1-s3-uri-style"
@@ -460,7 +464,7 @@ Option constants
#define CFGOPT_TYPE "type"
STRING_DECLARE(CFGOPT_TYPE_STR);
-#define CFG_OPTION_TOTAL 203
+#define CFG_OPTION_TOTAL 205
/***********************************************************************************************************************************
Command enum
@@ -672,8 +676,10 @@ typedef enum
cfgOptRepoS3Host,
cfgOptRepoS3Key,
cfgOptRepoS3KeySecret,
+ cfgOptRepoS3KeyType,
cfgOptRepoS3Port,
cfgOptRepoS3Region,
+ cfgOptRepoS3Role,
cfgOptRepoS3Token,
cfgOptRepoS3UriStyle,
cfgOptRepoS3VerifyTls,
diff --git a/src/config/define.auto.c b/src/config/define.auto.c
index 1196e3ec1..28d2e38b9 100644
--- a/src/config/define.auto.c
+++ b/src/config/define.auto.c
@@ -4549,8 +4549,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
- cfgDefOptRepoType,
- "s3"
+ cfgDefOptRepoS3KeyType,
+ "shared"
)
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
@@ -4599,12 +4599,74 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
+ CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
+ (
+ cfgDefOptRepoS3KeyType,
+ "shared"
+ )
+
+ CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
+ )
+ )
+
+ // -----------------------------------------------------------------------------------------------------------------------------
+ CFGDEFDATA_OPTION
+ (
+ CFGDEFDATA_OPTION_NAME("repo-s3-key-type")
+ CFGDEFDATA_OPTION_REQUIRED(true)
+ CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
+ CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString)
+ CFGDEFDATA_OPTION_INTERNAL(false)
+
+ CFGDEFDATA_OPTION_INDEX_TOTAL(1)
+ CFGDEFDATA_OPTION_SECURE(false)
+
+ CFGDEFDATA_OPTION_HELP_SECTION("repository")
+ CFGDEFDATA_OPTION_HELP_SUMMARY("S3 repository key type.")
+ CFGDEFDATA_OPTION_HELP_DESCRIPTION
+ (
+ "The following types are supported:\n"
+ "\n"
+ "* shared - Shared keys\n"
+ "* auto - Automatically retrieve temporary credentials"
+ )
+
+ CFGDEFDATA_OPTION_COMMAND_LIST
+ (
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoCreate)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoGet)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoLs)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoPut)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoRm)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStart)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStop)
+ )
+
+ CFGDEFDATA_OPTION_OPTIONAL_LIST
+ (
+ CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
+ (
+ "shared",
+ "auto"
+ )
+
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
cfgDefOptRepoType,
"s3"
)
+ CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("shared")
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
)
)
@@ -4715,6 +4777,58 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
)
)
+ // -----------------------------------------------------------------------------------------------------------------------------
+ CFGDEFDATA_OPTION
+ (
+ CFGDEFDATA_OPTION_NAME("repo-s3-role")
+ CFGDEFDATA_OPTION_REQUIRED(false)
+ CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
+ CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString)
+ CFGDEFDATA_OPTION_INTERNAL(false)
+
+ CFGDEFDATA_OPTION_INDEX_TOTAL(1)
+ CFGDEFDATA_OPTION_SECURE(false)
+
+ CFGDEFDATA_OPTION_HELP_SECTION("repository")
+ CFGDEFDATA_OPTION_HELP_SUMMARY("S3 repository role.")
+ CFGDEFDATA_OPTION_HELP_DESCRIPTION
+ (
+ "AWS role used to retrieve temporary credentials when repo-s3-key-type=auto."
+ )
+
+ CFGDEFDATA_OPTION_COMMAND_LIST
+ (
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoCreate)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoGet)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoLs)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoPut)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoRm)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStart)
+ CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStop)
+ )
+
+ CFGDEFDATA_OPTION_OPTIONAL_LIST
+ (
+ CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
+ (
+ cfgDefOptRepoS3KeyType,
+ "auto"
+ )
+
+ CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
+ )
+ )
+
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
@@ -4759,8 +4873,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
- cfgDefOptRepoType,
- "s3"
+ cfgDefOptRepoS3KeyType,
+ "shared"
)
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
diff --git a/src/config/define.auto.h b/src/config/define.auto.h
index 3f643de1d..c3b0be96e 100644
--- a/src/config/define.auto.h
+++ b/src/config/define.auto.h
@@ -146,8 +146,10 @@ typedef enum
cfgDefOptRepoS3Host,
cfgDefOptRepoS3Key,
cfgDefOptRepoS3KeySecret,
+ cfgDefOptRepoS3KeyType,
cfgDefOptRepoS3Port,
cfgDefOptRepoS3Region,
+ cfgDefOptRepoS3Role,
cfgDefOptRepoS3Token,
cfgDefOptRepoS3UriStyle,
cfgDefOptRepoS3VerifyTls,
diff --git a/src/config/parse.auto.c b/src/config/parse.auto.c
index 2bd75772f..b8d61910d 100644
--- a/src/config/parse.auto.c
+++ b/src/config/parse.auto.c
@@ -2376,6 +2376,18 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_DEPRECATE_FLAG | cfgOptRepoS3KeySecret,
},
+ // repo-s3-key-type option
+ // -----------------------------------------------------------------------------------------------------------------------------
+ {
+ .name = CFGOPT_REPO1_S3_KEY_TYPE,
+ .has_arg = required_argument,
+ .val = PARSE_OPTION_FLAG | cfgOptRepoS3KeyType,
+ },
+ {
+ .name = "reset-" CFGOPT_REPO1_S3_KEY_TYPE,
+ .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3KeyType,
+ },
+
// repo-s3-port option
// -----------------------------------------------------------------------------------------------------------------------------
{
@@ -2405,6 +2417,18 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_DEPRECATE_FLAG | cfgOptRepoS3Region,
},
+ // repo-s3-role option
+ // -----------------------------------------------------------------------------------------------------------------------------
+ {
+ .name = CFGOPT_REPO1_S3_ROLE,
+ .has_arg = required_argument,
+ .val = PARSE_OPTION_FLAG | cfgOptRepoS3Role,
+ },
+ {
+ .name = "reset-" CFGOPT_REPO1_S3_ROLE,
+ .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3Role,
+ },
+
// repo-s3-token option
// -----------------------------------------------------------------------------------------------------------------------------
{
@@ -2889,10 +2913,10 @@ static const ConfigOption optionResolveOrder[] =
cfgOptRepoS3CaPath,
cfgOptRepoS3Endpoint,
cfgOptRepoS3Host,
- cfgOptRepoS3Key,
- cfgOptRepoS3KeySecret,
+ cfgOptRepoS3KeyType,
cfgOptRepoS3Port,
cfgOptRepoS3Region,
+ cfgOptRepoS3Role,
cfgOptRepoS3Token,
cfgOptRepoS3UriStyle,
cfgOptRepoS3VerifyTls,
@@ -2900,4 +2924,6 @@ static const ConfigOption optionResolveOrder[] =
cfgOptTargetAction,
cfgOptTargetExclusive,
cfgOptTargetTimeline,
+ cfgOptRepoS3Key,
+ cfgOptRepoS3KeySecret,
};
diff --git a/src/storage/helper.c b/src/storage/helper.c
index a6db9d75f..8ef05d070 100644
--- a/src/storage/helper.c
+++ b/src/storage/helper.c
@@ -392,8 +392,10 @@ storageRepoGet(const String *type, bool write)
result = storageS3New(
cfgOptionStr(cfgOptRepoPath), write, storageRepoPathExpression, cfgOptionStr(cfgOptRepoS3Bucket), endPoint,
strEqZ(cfgOptionStr(cfgOptRepoS3UriStyle), STORAGE_S3_URI_STYLE_HOST) ? storageS3UriStyleHost : storageS3UriStylePath,
- cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key), cfgOptionStr(cfgOptRepoS3KeySecret),
- cfgOptionStrNull(cfgOptRepoS3Token), STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(),
+ cfgOptionStr(cfgOptRepoS3Region),
+ strEqZ(cfgOptionStr(cfgOptRepoS3KeyType), STORAGE_S3_KEY_TYPE_SHARED) ? storageS3KeyTypeShared : storageS3KeyTypeAuto,
+ cfgOptionStrNull(cfgOptRepoS3Key), cfgOptionStrNull(cfgOptRepoS3KeySecret), cfgOptionStrNull(cfgOptRepoS3Token),
+ cfgOptionStrNull(cfgOptRepoS3Role), STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(),
cfgOptionBool(cfgOptRepoS3VerifyTls), cfgOptionStrNull(cfgOptRepoS3CaFile), cfgOptionStrNull(cfgOptRepoS3CaPath));
}
else
diff --git a/src/storage/s3/storage.c b/src/storage/s3/storage.c
index bd3e8d78d..6ed4636e3 100644
--- a/src/storage/s3/storage.c
+++ b/src/storage/s3/storage.c
@@ -16,6 +16,7 @@ S3 Storage
#include "common/memContext.h"
#include "common/regExp.h"
#include "common/type/object.h"
+#include "common/type/json.h"
#include "common/type/xml.h"
#include "storage/s3/read.h"
#include "storage/s3/storage.intern.h"
@@ -66,6 +67,24 @@ STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix");
STRING_STATIC(S3_XML_TAG_QUIET_STR, "Quiet");
STRING_STATIC(S3_XML_TAG_SIZE_STR, "Size");
+/***********************************************************************************************************************************
+Constants for automatically fetching the current role and credentials
+
+Documentation for the response format is found at: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
+***********************************************************************************************************************************/
+STRING_STATIC(S3_CREDENTIAL_HOST_STR, "169.254.169.254");
+#define S3_CREDENTIAL_PORT 80
+#define S3_CREDENTIAL_URI "/latest/meta-data/iam/security-credentials"
+#define S3_CREDENTIAL_RENEW_SEC (5 * 60)
+
+VARIANT_STRDEF_STATIC(S3_JSON_TAG_ACCESS_KEY_ID_VAR, "AccessKeyId");
+VARIANT_STRDEF_STATIC(S3_JSON_TAG_CODE_VAR, "Code");
+VARIANT_STRDEF_STATIC(S3_JSON_TAG_EXPIRATION_VAR, "Expiration");
+VARIANT_STRDEF_STATIC(S3_JSON_TAG_SECRET_ACCESS_KEY_VAR, "SecretAccessKey");
+VARIANT_STRDEF_STATIC(S3_JSON_TAG_TOKEN_VAR, "Token");
+
+VARIANT_STRDEF_STATIC(S3_JSON_VALUE_SUCCESS_VAR, "Success");
+
/***********************************************************************************************************************************
AWS authentication v4 constants
***********************************************************************************************************************************/
@@ -93,14 +112,21 @@ struct StorageS3
const String *bucket; // Bucket to store data in
const String *region; // e.g. us-east-1
- const String *accessKey; // Access key
- const String *secretAccessKey; // Secret access key
- const String *securityToken; // Security token, if any
+ StorageS3KeyType keyType; // Key type (shared or temp)
+ String *accessKey; // Access key
+ String *secretAccessKey; // Secret access key
+ String *securityToken; // Security token, if any
size_t partSize; // Part size for multi-part upload
unsigned int deleteMax; // Maximum objects that can be deleted in one request
StorageS3UriStyle uriStyle; // Path or host style URIs
const String *bucketEndpoint; // Set to {bucket}.{endpoint}
+ // For retrieving temporary security credentials
+ HttpClient *credHttpClient; // HTTP client to service credential requests
+ const String *credHost; // Credentials host
+ const String *credRole; // Role to use for credential requests
+ time_t credExpirationTime; // Time the temporary credentials expire
+
// Current signing key and date it is valid for
const String *signingKeyDate; // Date of cached signing key (so we know when to regenerate)
const Buffer *signingKey; // Cached signing key
@@ -235,6 +261,22 @@ storageS3Auth(
/***********************************************************************************************************************************
Process S3 request
***********************************************************************************************************************************/
+// Helper to convert YYYY-MM-DDTHH:MM:SS.MSECZ format to time_t. This format is very nearly ISO-8601 except for the inclusion of
+// milliseconds, which are discarded here.
+static time_t
+storageS3CvtTime(const String *time)
+{
+ FUNCTION_TEST_BEGIN();
+ FUNCTION_TEST_PARAM(STRING, time);
+ FUNCTION_TEST_END();
+
+ FUNCTION_TEST_RETURN(
+ epochFromParts(
+ cvtZToInt(strZ(strSubN(time, 0, 4))), cvtZToInt(strZ(strSubN(time, 5, 2))),
+ cvtZToInt(strZ(strSubN(time, 8, 2))), cvtZToInt(strZ(strSubN(time, 11, 2))),
+ cvtZToInt(strZ(strSubN(time, 14, 2))), cvtZToInt(strZ(strSubN(time, 17, 2))), 0));
+}
+
HttpRequest *
storageS3RequestAsync(StorageS3 *this, const String *verb, const String *uri, StorageS3RequestAsyncParam param)
{
@@ -273,6 +315,98 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *uri, St
if (this->uriStyle == storageS3UriStylePath)
uri = strNewFmt("/%s%s", strZ(this->bucket), strZ(uri));
+ // If temp crendentials will be expiring soon then renew them
+ if (this->keyType == storageS3KeyTypeAuto && (this->credExpirationTime - time(NULL)) < S3_CREDENTIAL_RENEW_SEC)
+ {
+ // Set content-length and host headers
+ HttpHeader *credHeader = httpHeaderNew(NULL);
+ httpHeaderAdd(credHeader, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
+ httpHeaderAdd(credHeader, HTTP_HEADER_HOST_STR, this->credHost);
+
+ // If the role was not set explicitly or retrieved previously then retrieve it
+ if (this->credRole == NULL)
+ {
+ // Request the role
+ HttpRequest *request = httpRequestNewP(
+ this->credHttpClient, HTTP_VERB_GET_STR, STRDEF(S3_CREDENTIAL_URI), .header = credHeader);
+ HttpResponse *response = httpRequestResponse(request, true);
+
+ // Not found likely means no role is associated with this instance
+ if (httpResponseCode(response) == HTTP_RESPONSE_CODE_NOT_FOUND)
+ {
+ THROW(
+ ProtocolError,
+ "role to retrieve temporary credentials not found\n"
+ "HINT: is a valid IAM role associated with this instance?");
+ }
+ // Else an error that we can't handle
+ else if (!httpResponseCodeOk(response))
+ httpRequestError(request, response);
+
+ // Get role from the text response
+ MEM_CONTEXT_BEGIN(this->memContext)
+ {
+ this->credRole = strNewBuf(httpResponseContent(response));
+ }
+ MEM_CONTEXT_END();
+ }
+
+ // Retrieve the temp credentials
+ HttpRequest *request = httpRequestNewP(
+ this->credHttpClient, HTTP_VERB_GET_STR, strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(this->credRole)),
+ .header = credHeader);
+ HttpResponse *response = httpRequestResponse(request, true);
+
+ // Not found likely means the role is not valid
+ if (httpResponseCode(response) == HTTP_RESPONSE_CODE_NOT_FOUND)
+ {
+ THROW_FMT(
+ ProtocolError,
+ "role '%s' not found\n"
+ "HINT: is '%s' a valid IAM role associated with this instance?",
+ strZ(this->credRole), strZ(this->credRole));
+ }
+ // Else an error that we can't handle
+ else if (!httpResponseCodeOk(response))
+ httpRequestError(request, response);
+
+ // Free old credentials
+ strFree(this->accessKey);
+ strFree(this->secretAccessKey);
+ strFree(this->securityToken);
+
+ // Get credentials from the JSON response
+ KeyValue *credential = jsonToKv(strNewBuf(httpResponseContent(response)));
+
+ MEM_CONTEXT_BEGIN(this->memContext)
+ {
+ // Check the code field for errors
+ const Variant *code = kvGetDefault(credential, S3_JSON_TAG_CODE_VAR, VARSTRDEF("code field is missing"));
+ CHECK(code != NULL);
+
+ if (!varEq(code, S3_JSON_VALUE_SUCCESS_VAR))
+ THROW_FMT(FormatError, "unable to retrieve temporary credentials: %s", strZ(varStr(code)));
+
+ // Make sure the required values are present
+ CHECK(kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR) != NULL);
+ CHECK(kvGet(credential, S3_JSON_TAG_SECRET_ACCESS_KEY_VAR) != NULL);
+ CHECK(kvGet(credential, S3_JSON_TAG_TOKEN_VAR) != NULL);
+
+ // Copy credentials
+ this->accessKey = strDup(varStr(kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR)));
+ this->secretAccessKey = strDup(varStr(kvGet(credential, S3_JSON_TAG_SECRET_ACCESS_KEY_VAR)));
+ this->securityToken = strDup(varStr(kvGet(credential, S3_JSON_TAG_TOKEN_VAR)));
+ }
+ MEM_CONTEXT_END();
+
+ // Update expiration time
+ CHECK(kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR) != NULL);
+ this->credExpirationTime = storageS3CvtTime(varStr(kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR)));
+
+ // Reset the signing key date so the signing key gets regenerated
+ this->signingKeyDate = YYYYMMDD_STR;
+ }
+
// Generate authorization header
storageS3Auth(
this, verb, httpUriEncode(uri, true), param.query, storageS3DateTime(time(NULL)), requestHeader,
@@ -530,22 +664,6 @@ typedef struct StorageS3InfoListData
void *callbackData; // User-supplied callback data
} StorageS3InfoListData;
-// Helper to convert YYYY-MM-DDTHH:MM:SS.MSECZ format to time_t. This format is very nearly ISO-8601 except for the inclusion of
-// milliseconds which are discarded here.
-static time_t
-storageS3CvtTime(const String *time)
-{
- FUNCTION_TEST_BEGIN();
- FUNCTION_TEST_PARAM(STRING, time);
- FUNCTION_TEST_END();
-
- FUNCTION_TEST_RETURN(
- epochFromParts(
- cvtZToInt(strZ(strSubN(time, 0, 4))), cvtZToInt(strZ(strSubN(time, 5, 2))),
- cvtZToInt(strZ(strSubN(time, 8, 2))), cvtZToInt(strZ(strSubN(time, 11, 2))),
- cvtZToInt(strZ(strSubN(time, 14, 2))), cvtZToInt(strZ(strSubN(time, 17, 2))), 0));
-}
-
static void
storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *name, StorageType type, const XmlNode *xml)
{
@@ -841,9 +959,9 @@ static const StorageInterface storageInterfaceS3 =
Storage *
storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
- const String *endPoint, StorageS3UriStyle uriStyle, const String *region, const String *accessKey,
- const String *secretAccessKey, const String *securityToken, size_t partSize, const String *host, unsigned int port,
- TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
+ const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
+ const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host,
+ unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@@ -853,9 +971,11 @@ storageS3New(
FUNCTION_LOG_PARAM(STRING, endPoint);
FUNCTION_LOG_PARAM(ENUM, uriStyle);
FUNCTION_LOG_PARAM(STRING, region);
+ FUNCTION_LOG_PARAM(ENUM, keyType);
FUNCTION_TEST_PARAM(STRING, accessKey);
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
FUNCTION_TEST_PARAM(STRING, securityToken);
+ FUNCTION_TEST_PARAM(STRING, credRole);
FUNCTION_LOG_PARAM(SIZE, partSize);
FUNCTION_LOG_PARAM(STRING, host);
FUNCTION_LOG_PARAM(UINT, port);
@@ -869,8 +989,9 @@ storageS3New(
ASSERT(bucket != NULL);
ASSERT(endPoint != NULL);
ASSERT(region != NULL);
- ASSERT(accessKey != NULL);
- ASSERT(secretAccessKey != NULL);
+ ASSERT(
+ (keyType == storageS3KeyTypeShared && accessKey != NULL && secretAccessKey != NULL) ||
+ (keyType == storageS3KeyTypeAuto && accessKey == NULL && secretAccessKey == NULL && securityToken == NULL));
ASSERT(partSize != 0);
Storage *this = NULL;
@@ -885,6 +1006,7 @@ storageS3New(
.interface = storageInterfaceS3,
.bucket = strDup(bucket),
.region = strDup(region),
+ .keyType = keyType,
.accessKey = strDup(accessKey),
.secretAccessKey = strDup(secretAccessKey),
.securityToken = strDup(securityToken),
@@ -893,6 +1015,8 @@ storageS3New(
.uriStyle = uriStyle,
.bucketEndpoint = uriStyle == storageS3UriStyleHost ?
strNewFmt("%s.%s", strZ(bucket), strZ(endPoint)) : strDup(endPoint),
+ .credHost = S3_CREDENTIAL_HOST_STR,
+ .credRole = strDup(credRole),
// Force the signing key to be generated on the first run
.signingKeyDate = YYYYMMDD_STR,
@@ -905,6 +1029,10 @@ storageS3New(
driver->httpClient = httpClientNew(
tlsClientNew(sckClientNew(host, port, timeout), host, timeout, verifyPeer, caFile, caPath), timeout);
+ // Create the HTTP client used to retreive temporary security credentials
+ if (driver->keyType == storageS3KeyTypeAuto)
+ driver->credHttpClient = httpClientNew(sckClientNew(driver->credHost, S3_CREDENTIAL_PORT, timeout), timeout);
+
// Create list of redacted headers
driver->headerRedactList = strLstNew();
strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
diff --git a/src/storage/s3/storage.h b/src/storage/s3/storage.h
index 129c91fd5..6fdb6fbfe 100644
--- a/src/storage/s3/storage.h
+++ b/src/storage/s3/storage.h
@@ -12,6 +12,18 @@ Storage type
#define STORAGE_S3_TYPE "s3"
STRING_DECLARE(STORAGE_S3_TYPE_STR);
+/***********************************************************************************************************************************
+Key type
+***********************************************************************************************************************************/
+typedef enum
+{
+ storageS3KeyTypeShared,
+ storageS3KeyTypeAuto,
+} StorageS3KeyType;
+
+#define STORAGE_S3_KEY_TYPE_SHARED "shared"
+#define STORAGE_S3_KEY_TYPE_AUTO "auto"
+
/***********************************************************************************************************************************
URI style
***********************************************************************************************************************************/
@@ -34,8 +46,8 @@ Constructors
***********************************************************************************************************************************/
Storage *storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
- const String *endPoint, StorageS3UriStyle uriStyle, const String *region, const String *accessKey,
- const String *secretAccessKey, const String *securityToken, size_t partSize, const String *host, unsigned int port,
- TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
+ const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
+ const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host,
+ unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
#endif
diff --git a/test/src/module/command/helpTest.c b/test/src/module/command/helpTest.c
index bfc7090e3..c23ec3edf 100644
--- a/test/src/module/command/helpTest.c
+++ b/test/src/module/command/helpTest.c
@@ -218,8 +218,10 @@ testRun(void)
" --repo-s3-host s3 repository host\n"
" --repo-s3-key s3 repository access key\n"
" --repo-s3-key-secret s3 repository secret access key\n"
+ " --repo-s3-key-type s3 repository key type [default=shared]\n"
" --repo-s3-port s3 repository port [default=443]\n"
" --repo-s3-region s3 repository region\n"
+ " --repo-s3-role s3 repository role\n"
" --repo-s3-token s3 repository security token\n"
" --repo-s3-uri-style s3 URI Style [default=host]\n"
" --repo-s3-verify-tls verify S3 server certificate [default=y]\n"
diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c
index 431e2f029..5636ca43c 100644
--- a/test/src/module/storage/s3Test.c
+++ b/test/src/module/storage/s3Test.c
@@ -34,26 +34,43 @@ typedef struct TestRequestParam
static void
testRequest(IoWrite *write, Storage *s3, const char *verb, const char *uri, TestRequestParam param)
{
- // Get S3 driver
- StorageS3 *driver = (StorageS3 *)storageDriver(s3);
+ // Get security token from param
+ const char *securityToken = param.securityToken == NULL ? NULL : param.securityToken;
- // Add verb, uri, version, user-agent, and authorization string
- String *request = strNewFmt(
- "%s %s HTTP/1.1\r\n"
- "user-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n"
- "authorization:AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,"
- "SignedHeaders=content-length",
- verb, uri);
+ // If s3 storage is set then get the driver
+ StorageS3 *driver = NULL;
- if (param.content != NULL)
- strCatZ(request, ";content-md5");
+ if (s3 != NULL)
+ {
+ driver = (StorageS3 *)storageDriver(s3);
- strCatZ(request, ";host;x-amz-content-sha256;x-amz-date");
+ // Also update the security token if it is not already set
+ if (param.securityToken == NULL)
+ securityToken = strZNull(driver->securityToken);
+ }
- if (driver->securityToken != NULL)
- strCatZ(request, ";x-amz-security-token");
+ // Add request
+ String *request = strNewFmt("%s %s HTTP/1.1\r\nuser-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n", verb, uri);
- strCatZ(request, ",Signature=????????????????????????????????????????????????????????????????\r\n");
+ // Add authorization header when s3 service
+ if (s3 != NULL)
+ {
+ strCatFmt(
+ request,
+ "authorization:AWS4-HMAC-SHA256 Credential=%s/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,"
+ "SignedHeaders=content-length",
+ param.accessKey == NULL ? strZ(driver->accessKey) : param.accessKey);
+
+ if (param.content != NULL)
+ strCatZ(request, ";content-md5");
+
+ strCatZ(request, ";host;x-amz-content-sha256;x-amz-date");
+
+ if (securityToken != NULL)
+ strCatZ(request, ";x-amz-security-token");
+
+ strCatZ(request, ",Signature=????????????????????????????????????????????????????????????????\r\n");
+ }
// Add content-length
strCatFmt(request, "content-length:%zu\r\n", param.content != NULL ? strlen(param.content) : 0);
@@ -67,21 +84,31 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *uri, Test
}
// Add host
- if (driver->uriStyle == storageS3UriStyleHost)
+ if (s3 != NULL)
+ {
+ if (driver->uriStyle == storageS3UriStyleHost)
strCatFmt(request, "host:bucket." S3_TEST_HOST "\r\n");
else
strCatFmt(request, "host:" S3_TEST_HOST "\r\n");
+ }
+ else
+ strCatFmt(request, "host:%s\r\n", strZ(hrnServerHost()));
- // Add content sha256 and date
- strCatFmt(
- request,
- "x-amz-content-sha256:%s\r\n"
- "x-amz-date:????????T??????Z" "\r\n",
- param.content == NULL ? HASH_TYPE_SHA256_ZERO : strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR,
- BUFSTRZ(param.content)))));
+ // Add content checksum and date if s3 service
+ if (s3 != NULL)
+ {
+ // Add content sha256 and date
+ strCatFmt(
+ request,
+ "x-amz-content-sha256:%s\r\n"
+ "x-amz-date:????????T??????Z" "\r\n",
+ param.content == NULL ? HASH_TYPE_SHA256_ZERO : strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR,
+ BUFSTRZ(param.content)))));
- if (driver->securityToken != NULL)
- strCatFmt(request, "x-amz-security-token:%s\r\n", strZ(driver->securityToken));
+ // Add security token
+ if (securityToken != NULL)
+ strCatFmt(request, "x-amz-security-token:%s\r\n", securityToken);
+ }
// Add final \r\n
strCatZ(request, "\r\n");
@@ -100,6 +127,7 @@ typedef struct TestResponseParam
{
VAR_PARAM_HEADER;
unsigned int code;
+ const char *http;
const char *header;
const char *content;
} TestResponseParam;
@@ -114,7 +142,7 @@ testResponse(IoWrite *write, TestResponseParam param)
param.code = param.code == 0 ? 200 : param.code;
// Output header and code
- String *response = strNewFmt("HTTP/1.1 %u ", param.code);
+ String *response = strNewFmt("HTTP/%s %u ", param.http == NULL ? "1.1" : param.http, param.code);
// Add reason for some codes
switch (param.code)
@@ -155,6 +183,25 @@ testResponse(IoWrite *write, TestResponseParam param)
hrnServerScriptReply(write, response);
}
+/***********************************************************************************************************************************
+Format ISO-8601 date with - and :
+***********************************************************************************************************************************/
+static String *
+testS3DateTime(time_t time)
+{
+ FUNCTION_HARNESS_BEGIN();
+ FUNCTION_HARNESS_PARAM(TIME, time);
+ FUNCTION_HARNESS_END();
+
+ char buffer[21];
+
+ THROW_ON_SYS_ERROR(
+ strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%SZ", gmtime(&time)) != sizeof(buffer) - 1, AssertError,
+ "unable to format date");
+
+ FUNCTION_HARNESS_RESULT(STRING, strNew(buffer));
+}
+
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
@@ -170,12 +217,14 @@ testRun(void)
const String *endPoint = strNew("s3.amazonaws.com");
const String *host = hrnServerHost();
const unsigned int port = hrnServerPort(0);
+ const unsigned int authPort = hrnServerPort(1);
const String *accessKey = strNew("AKIAIOSFODNN7EXAMPLE");
const String *secretAccessKey = strNew("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
const String *securityToken = strNew(
"AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/q"
"kPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xV"
"qr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==");
+ const String *credRole = STRDEF("credrole");
// Config settings that are required for every test (without endpoint for special tests)
StringList *commonArgWithoutEndpointList = strLstNew();
@@ -318,10 +367,22 @@ testRun(void)
}
HARNESS_FORK_CHILD_END();
+ HARNESS_FORK_CHILD_BEGIN(0, true)
+ {
+ TEST_RESULT_VOID(
+ hrnServerRunP(
+ ioFdReadNew(strNew("auth server read"), HARNESS_FORK_CHILD_READ(), 5000), hrnServerProtocolSocket,
+ .port = authPort),
+ "auth server run");
+ }
+ HARNESS_FORK_CHILD_END();
+
HARNESS_FORK_PARENT_BEGIN()
{
IoWrite *service = hrnServerScriptBegin(
ioFdWriteNew(strNew("s3 client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000));
+ IoWrite *auth = hrnServerScriptBegin(
+ ioFdWriteNew(strNew("auth client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(1), 2000));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("config with keys, token, and host with custom port");
@@ -340,9 +401,6 @@ testRun(void)
TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
TEST_RESULT_BOOL(storageFeature(s3, storageFeatureCompress), false, "check compress feature");
- // Set partSize to a small value for testing
- driver->partSize = 16;
-
// Coverage for noop functions
// -----------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(storagePathSyncP(s3, strNew("path")), "path sync is a noop");
@@ -383,10 +441,145 @@ testRun(void)
TEST_RESULT_STR_Z(strNewBuf(storageGetP(storageNewReadP(s3, strNew("file0.txt")))), "", "get zero-length file");
+ // -----------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("switch to temp credentials");
+
+ hrnServerScriptClose(service);
+
+ argList = strLstDup(commonArgList);
+ strLstAdd(argList, strNewFmt("--" CFGOPT_REPO1_S3_HOST "=%s:%u", strZ(host), port));
+ strLstAdd(argList, strNewFmt("--" CFGOPT_REPO1_S3_ROLE "=%s", strZ(credRole)));
+ strLstAddZ(argList, "--" CFGOPT_REPO1_S3_KEY_TYPE "=" STORAGE_S3_KEY_TYPE_AUTO);
+ harnessCfgLoad(cfgCmdArchivePush, argList);
+
+ s3 = storageRepoGet(STORAGE_S3_TYPE_STR, true);
+ driver = (StorageS3 *)s3->driver;
+
+ TEST_RESULT_STR(s3->path, path, "check path");
+ TEST_RESULT_STR(driver->credRole, credRole, "check role");
+ TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
+ TEST_RESULT_BOOL(storageFeature(s3, storageFeatureCompress), false, "check compress feature");
+
+ // Set partSize to a small value for testing
+ driver->partSize = 16;
+
+ // Testing requires the auth http client to be redirected
+ driver->credHost = hrnServerHost();
+ driver->credHttpClient = httpClientNew(sckClientNew(host, authPort, 5000), 5000);
+
+ // Now that we have checked the role when set explicitly, null it out to make sure it is retrieved automatically
+ driver->credRole = NULL;
+
+ hrnServerScriptAccept(service);
+
+ // -----------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("error when retrieving role");
+
+ hrnServerScriptAccept(auth);
+
+ testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_URI);
+ testResponseP(auth, .http = "1.0", .code = 301);
+
+ hrnServerScriptClose(auth);
+
+ TEST_ERROR_FMT(
+ storageGetP(storageNewReadP(s3, strNew("file.txt"))), ProtocolError,
+ "HTTP request failed with 301:\n"
+ "*** URI/Query ***:\n"
+ "/latest/meta-data/iam/security-credentials\n"
+ "*** Request Headers ***:\n"
+ "content-length: 0\n"
+ "host: %s",
+ strZ(hrnServerHost()));
+
+ // -----------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("missing role");
+
+ hrnServerScriptAccept(auth);
+
+ testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_URI);
+ testResponseP(auth, .http = "1.0", .code = 404);
+
+ hrnServerScriptClose(auth);
+
+ TEST_ERROR(
+ storageGetP(storageNewReadP(s3, strNew("file.txt"))), ProtocolError,
+ "role to retrieve temporary credentials not found\n"
+ "HINT: is a valid IAM role associated with this instance?");
+
+ // -----------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("error when retrieving temp credentials");
+
+ hrnServerScriptAccept(auth);
+
+ testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_URI);
+ testResponseP(auth, .http = "1.0", .content = strZ(credRole));
+
+ hrnServerScriptClose(auth);
+ hrnServerScriptAccept(auth);
+
+ testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
+ testResponseP(auth, .http = "1.0", .code = 300);
+
+ hrnServerScriptClose(auth);
+
+ TEST_ERROR_FMT(
+ storageGetP(storageNewReadP(s3, strNew("file.txt"))), ProtocolError,
+ "HTTP request failed with 300:\n"
+ "*** URI/Query ***:\n"
+ "/latest/meta-data/iam/security-credentials/credrole\n"
+ "*** Request Headers ***:\n"
+ "content-length: 0\n"
+ "host: %s",
+ strZ(hrnServerHost()));
+
+ // -----------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("invalid temp credentials role");
+
+ hrnServerScriptAccept(auth);
+
+ testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
+ testResponseP(auth, .http = "1.0", .code = 404);
+
+ hrnServerScriptClose(auth);
+
+ TEST_ERROR_FMT(
+ storageGetP(storageNewReadP(s3, strNew("file.txt"))), ProtocolError,
+ "role '%s' not found\n"
+ "HINT: is '%s' a valid IAM role associated with this instance?",
+ strZ(credRole), strZ(credRole));
+
+ // -----------------------------------------------------------------------------------------------------------------
+ TEST_TITLE("invalid code when retrieving temp credentials");
+
+ hrnServerScriptAccept(auth);
+
+ testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
+ testResponseP(auth, .http = "1.0", .content = "{\"Code\":\"IAM role is not configured\"}");
+
+ hrnServerScriptClose(auth);
+
+ TEST_ERROR(
+ storageGetP(storageNewReadP(s3, strNew("file.txt"))), FormatError,
+ "unable to retrieve temporary credentials: IAM role is not configured");
+
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("non-404 error");
- testRequestP(service, s3, HTTP_VERB_GET, "/file.txt");
+ hrnServerScriptAccept(auth);
+
+ testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
+ testResponseP(
+ auth,
+ .content = strZ(
+ strNewFmt(
+ "{\"Code\":\"Success\",\"AccessKeyId\":\"x\",\"SecretAccessKey\":\"y\",\"Token\":\"z\""
+ ",\"Expiration\":\"%s\"}",
+ strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC - 1))))));
+
+ hrnServerScriptClose(auth);
+
+ testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .accessKey = "x", .securityToken = "z");
testResponseP(service, .code = 303, .content = "CONTENT");
StorageRead *read = NULL;
@@ -411,12 +604,33 @@ testRun(void)
"*** Response Content ***:\n"
"CONTENT")
+ // Check that temp credentials were set
+ TEST_RESULT_STR_Z(driver->accessKey, "x", "check access key");
+ TEST_RESULT_STR_Z(driver->secretAccessKey, "y", "check secret access key");
+ TEST_RESULT_STR_Z(driver->securityToken, "z", "check security token");
+
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in one part");
- testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD");
+ hrnServerScriptAccept(auth);
+
+ testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
+ testResponseP(
+ auth,
+ .content = strZ(
+ strNewFmt(
+ "{\"Code\":\"Success\",\"AccessKeyId\":\"xx\",\"SecretAccessKey\":\"yy\",\"Token\":\"zz\""
+ ",\"Expiration\":\"%s\"}",
+ strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC * 2))))));
+
+ hrnServerScriptClose(auth);
+
+ testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz");
testResponseP(service);
+ // Make a copy of the signing key to verify that it gets changed when the keys are updated
+ const Buffer *oldSigningKey = bufDup(driver->signingKey);
+
StorageWrite *write = NULL;
TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write");
TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("ABCD")), "write");
@@ -431,6 +645,17 @@ testRun(void)
TEST_RESULT_VOID(storageWriteS3Close(write->driver), "close file again");
+ // Check that temp credentials were changed
+ TEST_RESULT_STR_Z(driver->accessKey, "xx", "check access key");
+ TEST_RESULT_STR_Z(driver->secretAccessKey, "yy", "check secret access key");
+ TEST_RESULT_STR_Z(driver->securityToken, "zz", "check security token");
+
+ // Check that the signing key changed
+ TEST_RESULT_BOOL(bufEq(driver->signingKey, oldSigningKey), false, "signing key changed");
+
+ // Auth service no longer needed
+ hrnServerScriptEnd(auth);
+
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write zero-length file");