db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Automatically retrieve temporary S3 credentials on AWS instances.

Browse Source 
Automatically retrieve the role and temporary credentials for S3 when the AWS instance is associated with an IAM role. Credentials are automatically updated when they are <= 5 minutes from expiring.

Basic configuration is to set repo1-s3-key-type=auto. repo1-s3-role can be used to set a specific role, otherwise it will be retrieved automatically.
This commit is contained in:
David Steele 2020-08-25 10:38:49 -04:00 committed by GitHub
parent 1d2b8ae7bb
commit 851f2e814e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 674 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
build/lib/pgBackRestBuild/Config/Data.pm
View File

@ -227,12 +227,14 @@ use constant CFGOPT_REPO_AZURE_VERIFY_TLS => CFGDEF_RE
use constant CFGDEF_REPO_S3 => CFGDEF_PREFIX_REPO . '-s3';
use constant CFGOPT_REPO_S3_KEY => CFGDEF_REPO_S3 . '-key';
use constant CFGOPT_REPO_S3_KEY_SECRET => CFGDEF_REPO_S3 . '-key-secret';
use constant CFGOPT_REPO_S3_KEY_TYPE => CFGDEF_REPO_S3 . '-key-type';
use constant CFGOPT_REPO_S3_BUCKET => CFGDEF_REPO_S3 . '-bucket';
use constant CFGOPT_REPO_S3_CA_FILE => CFGDEF_REPO_S3 . '-ca-file';
use constant CFGOPT_REPO_S3_CA_PATH => CFGDEF_REPO_S3 . '-ca-path';
use constant CFGOPT_REPO_S3_ENDPOINT => CFGDEF_REPO_S3 . '-endpoint';
use constant CFGOPT_REPO_S3_HOST => CFGDEF_REPO_S3 . '-host';
use constant CFGOPT_REPO_S3_PORT => CFGDEF_REPO_S3 . '-port';
use constant CFGOPT_REPO_S3_ROLE => CFGDEF_REPO_S3 . '-role';
use constant CFGOPT_REPO_S3_REGION => CFGDEF_REPO_S3 . '-region';
use constant CFGOPT_REPO_S3_TOKEN => CFGDEF_REPO_S3 . '-token';
use constant CFGOPT_REPO_S3_URI_STYLE => CFGDEF_REPO_S3 . '-uri-style';
@ -1886,6 +1888,17 @@ my %hConfigDefine =
},
},
&CFGOPT_REPO_S3_KEY_TYPE =>
{
&CFGDEF_INHERIT => CFGOPT_REPO_S3_BUCKET,
&CFGDEF_DEFAULT => 'shared',
&CFGDEF_ALLOW_LIST =>
[
'shared',
'auto',
],
},
&CFGOPT_REPO_S3_KEY =>
{
&CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL,
@ -1896,8 +1909,8 @@ my %hConfigDefine =
&CFGDEF_REQUIRED => true,
&CFGDEF_DEPEND =>
{
&CFGDEF_DEPEND_OPTION => CFGOPT_REPO_TYPE,
&CFGDEF_DEPEND_LIST => [CFGOPTVAL_REPO_TYPE_S3],
&CFGDEF_DEPEND_OPTION => CFGOPT_REPO_S3_KEY_TYPE,
&CFGDEF_DEPEND_LIST => ['shared'],
},
&CFGDEF_NAME_ALT =>
{
@ -1979,6 +1992,17 @@ my %hConfigDefine =
},
},
&CFGOPT_REPO_S3_ROLE =>
{
&CFGDEF_INHERIT => CFGOPT_REPO_S3_BUCKET,
&CFGDEF_REQUIRED => false,
&CFGDEF_DEPEND =>
{
&CFGDEF_DEPEND_OPTION => CFGOPT_REPO_S3_KEY_TYPE,
&CFGDEF_DEPEND_LIST => ['auto'],
},
},
&CFGOPT_REPO_S3_TOKEN =>
{
&CFGDEF_INHERIT => CFGOPT_REPO_S3_KEY,

22
doc/xml/reference.xml
View File

@ -547,6 +547,19 @@
<example>wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY</example>
</config-key>
<!-- ======================================================================================================= -->
<config-key id="repo-s3-key-type" name="S3 Repository Key Type">
<summary>S3 repository key type.</summary>
<text>The following types are supported:
<ul>
<li><id>shared</id> - Shared keys</li>
<li><id>auto</id> - Automatically retrieve temporary credentials</li>
</ul></text>
<example>auto</example>
</config-key>
<!-- CONFIG - REPO SECTION - REPO-S3-TOKEN KEY -->
<config-key id="repo-s3-token" name="S3 Repository Security Token">
<summary>S3 repository security token.</summary>
@ -612,6 +625,15 @@
<example>9000</example>
</config-key>
<!-- ======================================================================================================= -->
<config-key id="repo-s3-role" name="S3 Repository Role">
<summary>S3 repository role.</summary>
<text>AWS role used to retrieve temporary credentials when <br-option>repo-s3-key-type=auto</br-option>.</text>
<example>authrole</example>
</config-key>
<!-- CONFIG - REPO SECTION - REPO-S3-REGION KEY -->
<config-key id="repo-s3-region" name="S3 Repository Region">
<summary>S3 repository region.</summary>

21
doc/xml/release.xml
View File

@ -41,6 +41,22 @@
</release-item>
</release-bug-list>
<release-feature-list>
<release-item>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-contributor id="stephen.frost"/>
<release-item-reviewer id="cynthia.shang"/>
<!-- Actually tester, but we don't have a tag for that yet -->
<release-item-reviewer id="david.youatt"/>
<release-item-reviewer id="ales.zeleny"/>
<release-item-reviewer id="jeanette.bromage"/>
</release-item-contributor-list>
<p>Automatically retrieve temporary <proper>S3</proper> credentials on <proper>AWS</proper> instances.</p>
</release-item>
</release-feature-list>
<release-improvement-list>
<release-item>
<p><postgres/> 13 beta3 support.</p>
@ -8840,6 +8856,11 @@
<contributor-id type="github">huseynsnmz</contributor-id>
</contributor>
<contributor id="jeanette.bromage">
<contributor-name-display>Jeanette Bromage</contributor-name-display>
<contributor-id type="github">JBromage</contributor-id>
</contributor>
<contributor id="keith.fiske">
<contributor-name-display>Keith Fiske</contributor-name-display>
<contributor-id type="github">keithf4</contributor-id>

4
doc/xml/user-guide.xml
View File

@ -2204,7 +2204,9 @@
<block-variable-replace key="s3-setup-create-bucket">y</block-variable-replace>
</block>
<p>A role should be created to run <backrest/> and the bucket permissions should be set as restrictively as possible. This sample <proper>Amazon S3</proper> policy will restrict all reads and writes to the bucket and repository path.</p>
<p>A role should be created to run <backrest/> and the bucket permissions should be set as restrictively as possible. If the role is associated with an instance in <proper>AWS</proper> then <backrest/> will automatically retrieve temporary credentials when <br-option>repo1-s3-key-type=auto</br-option>, which means that keys do not need to be explicitly set in <file>{[backrest-config-demo]}</file>.</p>
<p>This sample <proper>Amazon S3</proper> policy will restrict all reads and writes to the bucket and repository path.</p>
<code-block title="Sample Amazon S3 Policy">
{

18
src/config/config.auto.c
View File

@ -461,8 +461,10 @@ STRING_EXTERN(CFGOPT_REPO1_S3_ENDPOINT_STR, CFGOPT_REPO1
STRING_EXTERN(CFGOPT_REPO1_S3_HOST_STR, CFGOPT_REPO1_S3_HOST);
STRING_EXTERN(CFGOPT_REPO1_S3_KEY_STR, CFGOPT_REPO1_S3_KEY);
STRING_EXTERN(CFGOPT_REPO1_S3_KEY_SECRET_STR, CFGOPT_REPO1_S3_KEY_SECRET);
STRING_EXTERN(CFGOPT_REPO1_S3_KEY_TYPE_STR, CFGOPT_REPO1_S3_KEY_TYPE);
STRING_EXTERN(CFGOPT_REPO1_S3_PORT_STR, CFGOPT_REPO1_S3_PORT);
STRING_EXTERN(CFGOPT_REPO1_S3_REGION_STR, CFGOPT_REPO1_S3_REGION);
STRING_EXTERN(CFGOPT_REPO1_S3_ROLE_STR, CFGOPT_REPO1_S3_ROLE);
STRING_EXTERN(CFGOPT_REPO1_S3_TOKEN_STR, CFGOPT_REPO1_S3_TOKEN);
STRING_EXTERN(CFGOPT_REPO1_S3_URI_STYLE_STR, CFGOPT_REPO1_S3_URI_STYLE);
STRING_EXTERN(CFGOPT_REPO1_S3_VERIFY_TLS_STR, CFGOPT_REPO1_S3_VERIFY_TLS);
@ -1916,6 +1918,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3KeySecret)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
CONFIG_OPTION_NAME(CFGOPT_REPO1_S3_KEY_TYPE)
CONFIG_OPTION_INDEX(0)
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3KeyType)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
@ -1932,6 +1942,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3Region)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
CONFIG_OPTION_NAME(CFGOPT_REPO1_S3_ROLE)
CONFIG_OPTION_INDEX(0)
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3Role)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(

8
src/config/config.auto.h
View File

@ -409,10 +409,14 @@ Option constants
STRING_DECLARE(CFGOPT_REPO1_S3_KEY_STR);
#define CFGOPT_REPO1_S3_KEY_SECRET "repo1-s3-key-secret"
STRING_DECLARE(CFGOPT_REPO1_S3_KEY_SECRET_STR);
#define CFGOPT_REPO1_S3_KEY_TYPE "repo1-s3-key-type"
STRING_DECLARE(CFGOPT_REPO1_S3_KEY_TYPE_STR);
#define CFGOPT_REPO1_S3_PORT "repo1-s3-port"
STRING_DECLARE(CFGOPT_REPO1_S3_PORT_STR);
#define CFGOPT_REPO1_S3_REGION "repo1-s3-region"
STRING_DECLARE(CFGOPT_REPO1_S3_REGION_STR);
#define CFGOPT_REPO1_S3_ROLE "repo1-s3-role"
STRING_DECLARE(CFGOPT_REPO1_S3_ROLE_STR);
#define CFGOPT_REPO1_S3_TOKEN "repo1-s3-token"
STRING_DECLARE(CFGOPT_REPO1_S3_TOKEN_STR);
#define CFGOPT_REPO1_S3_URI_STYLE "repo1-s3-uri-style"
@ -460,7 +464,7 @@ Option constants
#define CFGOPT_TYPE "type"
STRING_DECLARE(CFGOPT_TYPE_STR);
#define CFG_OPTION_TOTAL 203
#define CFG_OPTION_TOTAL 205
/***********************************************************************************************************************************
Command enum
@ -672,8 +676,10 @@ typedef enum
cfgOptRepoS3Host,
cfgOptRepoS3Key,
cfgOptRepoS3KeySecret,
cfgOptRepoS3KeyType,
cfgOptRepoS3Port,
cfgOptRepoS3Region,
cfgOptRepoS3Role,
cfgOptRepoS3Token,
cfgOptRepoS3UriStyle,
cfgOptRepoS3VerifyTls,

122
src/config/define.auto.c
View File

@ -4549,8 +4549,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
cfgDefOptRepoType,
"s3"
cfgDefOptRepoS3KeyType,
"shared"
)
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
@ -4599,12 +4599,74 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
cfgDefOptRepoS3KeyType,
"shared"
)
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
CFGDEFDATA_OPTION_NAME("repo-s3-key-type")
CFGDEFDATA_OPTION_REQUIRED(true)
CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString)
CFGDEFDATA_OPTION_INTERNAL(false)
CFGDEFDATA_OPTION_INDEX_TOTAL(1)
CFGDEFDATA_OPTION_SECURE(false)
CFGDEFDATA_OPTION_HELP_SECTION("repository")
CFGDEFDATA_OPTION_HELP_SUMMARY("S3 repository key type.")
CFGDEFDATA_OPTION_HELP_DESCRIPTION
(
"The following types are supported:\n"
"\n"
"* shared - Shared keys\n"
"* auto - Automatically retrieve temporary credentials"
)
CFGDEFDATA_OPTION_COMMAND_LIST
(
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoLs)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoPut)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoRm)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStart)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStop)
)
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
(
"shared",
"auto"
)
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
cfgDefOptRepoType,
"s3"
)
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("shared")
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
)
)
@ -4715,6 +4777,58 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
CFGDEFDATA_OPTION_NAME("repo-s3-role")
CFGDEFDATA_OPTION_REQUIRED(false)
CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString)
CFGDEFDATA_OPTION_INTERNAL(false)
CFGDEFDATA_OPTION_INDEX_TOTAL(1)
CFGDEFDATA_OPTION_SECURE(false)
CFGDEFDATA_OPTION_HELP_SECTION("repository")
CFGDEFDATA_OPTION_HELP_SUMMARY("S3 repository role.")
CFGDEFDATA_OPTION_HELP_DESCRIPTION
(
"AWS role used to retrieve temporary credentials when repo-s3-key-type=auto."
)
CFGDEFDATA_OPTION_COMMAND_LIST
(
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoLs)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoPut)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoRm)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStart)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStop)
)
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
cfgDefOptRepoS3KeyType,
"auto"
)
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
@ -4759,8 +4873,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
cfgDefOptRepoType,
"s3"
cfgDefOptRepoS3KeyType,
"shared"
)
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")

2
src/config/define.auto.h
View File

@ -146,8 +146,10 @@ typedef enum
cfgDefOptRepoS3Host,
cfgDefOptRepoS3Key,
cfgDefOptRepoS3KeySecret,
cfgDefOptRepoS3KeyType,
cfgDefOptRepoS3Port,
cfgDefOptRepoS3Region,
cfgDefOptRepoS3Role,
cfgDefOptRepoS3Token,
cfgDefOptRepoS3UriStyle,
cfgDefOptRepoS3VerifyTls,

30
src/config/parse.auto.c
View File

@ -2376,6 +2376,18 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_DEPRECATE_FLAG | cfgOptRepoS3KeySecret,
},
// repo-s3-key-type option
// -----------------------------------------------------------------------------------------------------------------------------
{
.name = CFGOPT_REPO1_S3_KEY_TYPE,
.has_arg = required_argument,
.val = PARSE_OPTION_FLAG | cfgOptRepoS3KeyType,
},
{
.name = "reset-" CFGOPT_REPO1_S3_KEY_TYPE,
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3KeyType,
},
// repo-s3-port option
// -----------------------------------------------------------------------------------------------------------------------------
{
@ -2405,6 +2417,18 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_DEPRECATE_FLAG | cfgOptRepoS3Region,
},
// repo-s3-role option
// -----------------------------------------------------------------------------------------------------------------------------
{
.name = CFGOPT_REPO1_S3_ROLE,
.has_arg = required_argument,
.val = PARSE_OPTION_FLAG | cfgOptRepoS3Role,
},
{
.name = "reset-" CFGOPT_REPO1_S3_ROLE,
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3Role,
},
// repo-s3-token option
// -----------------------------------------------------------------------------------------------------------------------------
{
@ -2889,10 +2913,10 @@ static const ConfigOption optionResolveOrder[] =
cfgOptRepoS3CaPath,
cfgOptRepoS3Endpoint,
cfgOptRepoS3Host,
cfgOptRepoS3Key,
cfgOptRepoS3KeySecret,
cfgOptRepoS3KeyType,
cfgOptRepoS3Port,
cfgOptRepoS3Region,
cfgOptRepoS3Role,
cfgOptRepoS3Token,
cfgOptRepoS3UriStyle,
cfgOptRepoS3VerifyTls,
@ -2900,4 +2924,6 @@ static const ConfigOption optionResolveOrder[] =
cfgOptTargetAction,
cfgOptTargetExclusive,
cfgOptTargetTimeline,
cfgOptRepoS3Key,
cfgOptRepoS3KeySecret,
};

6
src/storage/helper.c
View File

@ -392,8 +392,10 @@ storageRepoGet(const String *type, bool write)
result = storageS3New(
cfgOptionStr(cfgOptRepoPath), write, storageRepoPathExpression, cfgOptionStr(cfgOptRepoS3Bucket), endPoint,
strEqZ(cfgOptionStr(cfgOptRepoS3UriStyle), STORAGE_S3_URI_STYLE_HOST) ? storageS3UriStyleHost : storageS3UriStylePath,
cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key), cfgOptionStr(cfgOptRepoS3KeySecret),
cfgOptionStrNull(cfgOptRepoS3Token), STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(),
cfgOptionStr(cfgOptRepoS3Region),
strEqZ(cfgOptionStr(cfgOptRepoS3KeyType), STORAGE_S3_KEY_TYPE_SHARED) ? storageS3KeyTypeShared : storageS3KeyTypeAuto,
cfgOptionStrNull(cfgOptRepoS3Key), cfgOptionStrNull(cfgOptRepoS3KeySecret), cfgOptionStrNull(cfgOptRepoS3Token),
cfgOptionStrNull(cfgOptRepoS3Role), STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(),
cfgOptionBool(cfgOptRepoS3VerifyTls), cfgOptionStrNull(cfgOptRepoS3CaFile), cfgOptionStrNull(cfgOptRepoS3CaPath));
}
else

176
src/storage/s3/storage.c
View File

@ -16,6 +16,7 @@ S3 Storage
#include "common/memContext.h"
#include "common/regExp.h"
#include "common/type/object.h"
#include "common/type/json.h"
#include "common/type/xml.h"
#include "storage/s3/read.h"
#include "storage/s3/storage.intern.h"
@ -66,6 +67,24 @@ STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix");
STRING_STATIC(S3_XML_TAG_QUIET_STR, "Quiet");
STRING_STATIC(S3_XML_TAG_SIZE_STR, "Size");
/***********************************************************************************************************************************
Constants for automatically fetching the current role and credentials
Documentation for the response format is found at: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
***********************************************************************************************************************************/
STRING_STATIC(S3_CREDENTIAL_HOST_STR, "169.254.169.254");
#define S3_CREDENTIAL_PORT 80
#define S3_CREDENTIAL_URI "/latest/meta-data/iam/security-credentials"
#define S3_CREDENTIAL_RENEW_SEC (5 * 60)
VARIANT_STRDEF_STATIC(S3_JSON_TAG_ACCESS_KEY_ID_VAR, "AccessKeyId");
VARIANT_STRDEF_STATIC(S3_JSON_TAG_CODE_VAR, "Code");
VARIANT_STRDEF_STATIC(S3_JSON_TAG_EXPIRATION_VAR, "Expiration");
VARIANT_STRDEF_STATIC(S3_JSON_TAG_SECRET_ACCESS_KEY_VAR, "SecretAccessKey");
VARIANT_STRDEF_STATIC(S3_JSON_TAG_TOKEN_VAR, "Token");
VARIANT_STRDEF_STATIC(S3_JSON_VALUE_SUCCESS_VAR, "Success");
/***********************************************************************************************************************************
AWS authentication v4 constants
***********************************************************************************************************************************/
@ -93,14 +112,21 @@ struct StorageS3
const String *bucket; // Bucket to store data in
const String *region; // e.g. us-east-1
const String *accessKey; // Access key
const String *secretAccessKey; // Secret access key
const String *securityToken; // Security token, if any
StorageS3KeyType keyType; // Key type (shared or temp)
String *accessKey; // Access key
String *secretAccessKey; // Secret access key
String *securityToken; // Security token, if any
size_t partSize; // Part size for multi-part upload
unsigned int deleteMax; // Maximum objects that can be deleted in one request
StorageS3UriStyle uriStyle; // Path or host style URIs
const String *bucketEndpoint; // Set to {bucket}.{endpoint}
// For retrieving temporary security credentials
HttpClient *credHttpClient; // HTTP client to service credential requests
const String *credHost; // Credentials host
const String *credRole; // Role to use for credential requests
time_t credExpirationTime; // Time the temporary credentials expire
// Current signing key and date it is valid for
const String *signingKeyDate; // Date of cached signing key (so we know when to regenerate)
const Buffer *signingKey; // Cached signing key
@ -235,6 +261,22 @@ storageS3Auth(
/***********************************************************************************************************************************
Process S3 request
***********************************************************************************************************************************/
// Helper to convert YYYY-MM-DDTHH:MM:SS.MSECZ format to time_t. This format is very nearly ISO-8601 except for the inclusion of
// milliseconds, which are discarded here.
static time_t
storageS3CvtTime(const String *time)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, time);
FUNCTION_TEST_END();
FUNCTION_TEST_RETURN(
epochFromParts(
cvtZToInt(strZ(strSubN(time, 0, 4))), cvtZToInt(strZ(strSubN(time, 5, 2))),
cvtZToInt(strZ(strSubN(time, 8, 2))), cvtZToInt(strZ(strSubN(time, 11, 2))),
cvtZToInt(strZ(strSubN(time, 14, 2))), cvtZToInt(strZ(strSubN(time, 17, 2))), 0));
}
HttpRequest *
storageS3RequestAsync(StorageS3 *this, const String *verb, const String *uri, StorageS3RequestAsyncParam param)
{
@ -273,6 +315,98 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *uri, St
if (this->uriStyle == storageS3UriStylePath)
uri = strNewFmt("/%s%s", strZ(this->bucket), strZ(uri));
// If temp crendentials will be expiring soon then renew them
if (this->keyType == storageS3KeyTypeAuto && (this->credExpirationTime - time(NULL)) < S3_CREDENTIAL_RENEW_SEC)
{
// Set content-length and host headers
HttpHeader *credHeader = httpHeaderNew(NULL);
httpHeaderAdd(credHeader, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
httpHeaderAdd(credHeader, HTTP_HEADER_HOST_STR, this->credHost);
// If the role was not set explicitly or retrieved previously then retrieve it
if (this->credRole == NULL)
{
// Request the role
HttpRequest *request = httpRequestNewP(
this->credHttpClient, HTTP_VERB_GET_STR, STRDEF(S3_CREDENTIAL_URI), .header = credHeader);
HttpResponse *response = httpRequestResponse(request, true);
// Not found likely means no role is associated with this instance
if (httpResponseCode(response) == HTTP_RESPONSE_CODE_NOT_FOUND)
{
THROW(
ProtocolError,
"role to retrieve temporary credentials not found\n"
"HINT: is a valid IAM role associated with this instance?");
}
// Else an error that we can't handle
else if (!httpResponseCodeOk(response))
httpRequestError(request, response);
// Get role from the text response
MEM_CONTEXT_BEGIN(this->memContext)
{
this->credRole = strNewBuf(httpResponseContent(response));
}
MEM_CONTEXT_END();
}
// Retrieve the temp credentials
HttpRequest *request = httpRequestNewP(
this->credHttpClient, HTTP_VERB_GET_STR, strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(this->credRole)),
.header = credHeader);
HttpResponse *response = httpRequestResponse(request, true);
// Not found likely means the role is not valid
if (httpResponseCode(response) == HTTP_RESPONSE_CODE_NOT_FOUND)
{
THROW_FMT(
ProtocolError,
"role '%s' not found\n"
"HINT: is '%s' a valid IAM role associated with this instance?",
strZ(this->credRole), strZ(this->credRole));
}
// Else an error that we can't handle
else if (!httpResponseCodeOk(response))
httpRequestError(request, response);
// Free old credentials
strFree(this->accessKey);
strFree(this->secretAccessKey);
strFree(this->securityToken);
// Get credentials from the JSON response
KeyValue *credential = jsonToKv(strNewBuf(httpResponseContent(response)));
MEM_CONTEXT_BEGIN(this->memContext)
{
// Check the code field for errors
const Variant *code = kvGetDefault(credential, S3_JSON_TAG_CODE_VAR, VARSTRDEF("code field is missing"));
CHECK(code != NULL);
if (!varEq(code, S3_JSON_VALUE_SUCCESS_VAR))
THROW_FMT(FormatError, "unable to retrieve temporary credentials: %s", strZ(varStr(code)));
// Make sure the required values are present
CHECK(kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR) != NULL);
CHECK(kvGet(credential, S3_JSON_TAG_SECRET_ACCESS_KEY_VAR) != NULL);
CHECK(kvGet(credential, S3_JSON_TAG_TOKEN_VAR) != NULL);
// Copy credentials
this->accessKey = strDup(varStr(kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR)));
this->secretAccessKey = strDup(varStr(kvGet(credential, S3_JSON_TAG_SECRET_ACCESS_KEY_VAR)));
this->securityToken = strDup(varStr(kvGet(credential, S3_JSON_TAG_TOKEN_VAR)));
}
MEM_CONTEXT_END();
// Update expiration time
CHECK(kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR) != NULL);
this->credExpirationTime = storageS3CvtTime(varStr(kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR)));
// Reset the signing key date so the signing key gets regenerated
this->signingKeyDate = YYYYMMDD_STR;
}
// Generate authorization header
storageS3Auth(
this, verb, httpUriEncode(uri, true), param.query, storageS3DateTime(time(NULL)), requestHeader,
@ -530,22 +664,6 @@ typedef struct StorageS3InfoListData
void *callbackData; // User-supplied callback data
} StorageS3InfoListData;
// Helper to convert YYYY-MM-DDTHH:MM:SS.MSECZ format to time_t. This format is very nearly ISO-8601 except for the inclusion of
// milliseconds which are discarded here.
static time_t
storageS3CvtTime(const String *time)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, time);
FUNCTION_TEST_END();
FUNCTION_TEST_RETURN(
epochFromParts(
cvtZToInt(strZ(strSubN(time, 0, 4))), cvtZToInt(strZ(strSubN(time, 5, 2))),
cvtZToInt(strZ(strSubN(time, 8, 2))), cvtZToInt(strZ(strSubN(time, 11, 2))),
cvtZToInt(strZ(strSubN(time, 14, 2))), cvtZToInt(strZ(strSubN(time, 17, 2))), 0));
}
static void
storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *name, StorageType type, const XmlNode *xml)
{
@ -841,9 +959,9 @@ static const StorageInterface storageInterfaceS3 =
Storage *
storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, const String *accessKey,
const String *secretAccessKey, const String *securityToken, size_t partSize, const String *host, unsigned int port,
TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host,
unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@ -853,9 +971,11 @@ storageS3New(
FUNCTION_LOG_PARAM(STRING, endPoint);
FUNCTION_LOG_PARAM(ENUM, uriStyle);
FUNCTION_LOG_PARAM(STRING, region);
FUNCTION_LOG_PARAM(ENUM, keyType);
FUNCTION_TEST_PARAM(STRING, accessKey);
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
FUNCTION_TEST_PARAM(STRING, securityToken);
FUNCTION_TEST_PARAM(STRING, credRole);
FUNCTION_LOG_PARAM(SIZE, partSize);
FUNCTION_LOG_PARAM(STRING, host);
FUNCTION_LOG_PARAM(UINT, port);
@ -869,8 +989,9 @@ storageS3New(
ASSERT(bucket != NULL);
ASSERT(endPoint != NULL);
ASSERT(region != NULL);
ASSERT(accessKey != NULL);
ASSERT(secretAccessKey != NULL);
ASSERT(
(keyType == storageS3KeyTypeShared && accessKey != NULL && secretAccessKey != NULL) ||
(keyType == storageS3KeyTypeAuto && accessKey == NULL && secretAccessKey == NULL && securityToken == NULL));
ASSERT(partSize != 0);
Storage *this = NULL;
@ -885,6 +1006,7 @@ storageS3New(
.interface = storageInterfaceS3,
.bucket = strDup(bucket),
.region = strDup(region),
.keyType = keyType,
.accessKey = strDup(accessKey),
.secretAccessKey = strDup(secretAccessKey),
.securityToken = strDup(securityToken),
@ -893,6 +1015,8 @@ storageS3New(
.uriStyle = uriStyle,
.bucketEndpoint = uriStyle == storageS3UriStyleHost ?
strNewFmt("%s.%s", strZ(bucket), strZ(endPoint)) : strDup(endPoint),
.credHost = S3_CREDENTIAL_HOST_STR,
.credRole = strDup(credRole),
// Force the signing key to be generated on the first run
.signingKeyDate = YYYYMMDD_STR,
@ -905,6 +1029,10 @@ storageS3New(
driver->httpClient = httpClientNew(
tlsClientNew(sckClientNew(host, port, timeout), host, timeout, verifyPeer, caFile, caPath), timeout);
// Create the HTTP client used to retreive temporary security credentials
if (driver->keyType == storageS3KeyTypeAuto)
driver->credHttpClient = httpClientNew(sckClientNew(driver->credHost, S3_CREDENTIAL_PORT, timeout), timeout);
// Create list of redacted headers
driver->headerRedactList = strLstNew();
strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);

18
src/storage/s3/storage.h
View File

@ -12,6 +12,18 @@ Storage type
#define STORAGE_S3_TYPE "s3"
STRING_DECLARE(STORAGE_S3_TYPE_STR);
/***********************************************************************************************************************************
Key type
***********************************************************************************************************************************/
typedef enum
{
storageS3KeyTypeShared,
storageS3KeyTypeAuto,
} StorageS3KeyType;
#define STORAGE_S3_KEY_TYPE_SHARED "shared"
#define STORAGE_S3_KEY_TYPE_AUTO "auto"
/***********************************************************************************************************************************
URI style
***********************************************************************************************************************************/
@ -34,8 +46,8 @@ Constructors
***********************************************************************************************************************************/
Storage *storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, const String *accessKey,
const String *secretAccessKey, const String *securityToken, size_t partSize, const String *host, unsigned int port,
TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host,
unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
#endif

2
test/src/module/command/helpTest.c
View File

@ -218,8 +218,10 @@ testRun(void)
" --repo-s3-host s3 repository host\n"
" --repo-s3-key s3 repository access key\n"
" --repo-s3-key-secret s3 repository secret access key\n"
" --repo-s3-key-type s3 repository key type [default=shared]\n"
" --repo-s3-port s3 repository port [default=443]\n"
" --repo-s3-region s3 repository region\n"
" --repo-s3-role s3 repository role\n"
" --repo-s3-token s3 repository security token\n"
" --repo-s3-uri-style s3 URI Style [default=host]\n"
" --repo-s3-verify-tls verify S3 server certificate [default=y]\n"

287
test/src/module/storage/s3Test.c
View File

@ -34,26 +34,43 @@ typedef struct TestRequestParam
static void
testRequest(IoWrite *write, Storage *s3, const char *verb, const char *uri, TestRequestParam param)
{
// Get S3 driver
StorageS3 *driver = (StorageS3 *)storageDriver(s3);
// Get security token from param
const char *securityToken = param.securityToken == NULL ? NULL : param.securityToken;
// Add verb, uri, version, user-agent, and authorization string
String *request = strNewFmt(
"%s %s HTTP/1.1\r\n"
"user-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n"
"authorization:AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,"
"SignedHeaders=content-length",
verb, uri);
// If s3 storage is set then get the driver
StorageS3 *driver = NULL;
if (param.content != NULL)
strCatZ(request, ";content-md5");
if (s3 != NULL)
{
driver = (StorageS3 *)storageDriver(s3);
strCatZ(request, ";host;x-amz-content-sha256;x-amz-date");
// Also update the security token if it is not already set
if (param.securityToken == NULL)
securityToken = strZNull(driver->securityToken);
}
if (driver->securityToken != NULL)
strCatZ(request, ";x-amz-security-token");
// Add request
String *request = strNewFmt("%s %s HTTP/1.1\r\nuser-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n", verb, uri);
strCatZ(request, ",Signature=????????????????????????????????????????????????????????????????\r\n");
// Add authorization header when s3 service
if (s3 != NULL)
{
strCatFmt(
request,
"authorization:AWS4-HMAC-SHA256 Credential=%s/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,"
"SignedHeaders=content-length",
param.accessKey == NULL ? strZ(driver->accessKey) : param.accessKey);
if (param.content != NULL)
strCatZ(request, ";content-md5");
strCatZ(request, ";host;x-amz-content-sha256;x-amz-date");
if (securityToken != NULL)
strCatZ(request, ";x-amz-security-token");
strCatZ(request, ",Signature=????????????????????????????????????????????????????????????????\r\n");
}
// Add content-length
strCatFmt(request, "content-length:%zu\r\n", param.content != NULL ? strlen(param.content) : 0);
@ -67,21 +84,31 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *uri, Test
}
// Add host
if (driver->uriStyle == storageS3UriStyleHost)
if (s3 != NULL)
{
if (driver->uriStyle == storageS3UriStyleHost)
strCatFmt(request, "host:bucket." S3_TEST_HOST "\r\n");
else
strCatFmt(request, "host:" S3_TEST_HOST "\r\n");
}
else
strCatFmt(request, "host:%s\r\n", strZ(hrnServerHost()));
// Add content sha256 and date
strCatFmt(
request,
"x-amz-content-sha256:%s\r\n"
"x-amz-date:????????T??????Z" "\r\n",
param.content == NULL ? HASH_TYPE_SHA256_ZERO : strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR,
BUFSTRZ(param.content)))));
// Add content checksum and date if s3 service
if (s3 != NULL)
{
// Add content sha256 and date
strCatFmt(
request,
"x-amz-content-sha256:%s\r\n"
"x-amz-date:????????T??????Z" "\r\n",
param.content == NULL ? HASH_TYPE_SHA256_ZERO : strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR,
BUFSTRZ(param.content)))));
if (driver->securityToken != NULL)
strCatFmt(request, "x-amz-security-token:%s\r\n", strZ(driver->securityToken));
// Add security token
if (securityToken != NULL)
strCatFmt(request, "x-amz-security-token:%s\r\n", securityToken);
}
// Add final \r\n
strCatZ(request, "\r\n");
@ -100,6 +127,7 @@ typedef struct TestResponseParam
{
VAR_PARAM_HEADER;
unsigned int code;
const char *http;
const char *header;
const char *content;
} TestResponseParam;
@ -114,7 +142,7 @@ testResponse(IoWrite *write, TestResponseParam param)
param.code = param.code == 0 ? 200 : param.code;
// Output header and code
String *response = strNewFmt("HTTP/1.1 %u ", param.code);
String *response = strNewFmt("HTTP/%s %u ", param.http == NULL ? "1.1" : param.http, param.code);
// Add reason for some codes
switch (param.code)
@ -155,6 +183,25 @@ testResponse(IoWrite *write, TestResponseParam param)
hrnServerScriptReply(write, response);
}
/***********************************************************************************************************************************
Format ISO-8601 date with - and :
***********************************************************************************************************************************/
static String *
testS3DateTime(time_t time)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(TIME, time);
FUNCTION_HARNESS_END();
char buffer[21];
THROW_ON_SYS_ERROR(
strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%SZ", gmtime(&time)) != sizeof(buffer) - 1, AssertError,
"unable to format date");
FUNCTION_HARNESS_RESULT(STRING, strNew(buffer));
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
@ -170,12 +217,14 @@ testRun(void)
const String *endPoint = strNew("s3.amazonaws.com");
const String *host = hrnServerHost();
const unsigned int port = hrnServerPort(0);
const unsigned int authPort = hrnServerPort(1);
const String *accessKey = strNew("AKIAIOSFODNN7EXAMPLE");
const String *secretAccessKey = strNew("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
const String *securityToken = strNew(
"AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/q"
"kPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xV"
"qr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==");
const String *credRole = STRDEF("credrole");
// Config settings that are required for every test (without endpoint for special tests)
StringList *commonArgWithoutEndpointList = strLstNew();
@ -318,10 +367,22 @@ testRun(void)
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_CHILD_BEGIN(0, true)
{
TEST_RESULT_VOID(
hrnServerRunP(
ioFdReadNew(strNew("auth server read"), HARNESS_FORK_CHILD_READ(), 5000), hrnServerProtocolSocket,
.port = authPort),
"auth server run");
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
IoWrite *service = hrnServerScriptBegin(
ioFdWriteNew(strNew("s3 client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000));
IoWrite *auth = hrnServerScriptBegin(
ioFdWriteNew(strNew("auth client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(1), 2000));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("config with keys, token, and host with custom port");
@ -340,9 +401,6 @@ testRun(void)
TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
TEST_RESULT_BOOL(storageFeature(s3, storageFeatureCompress), false, "check compress feature");
// Set partSize to a small value for testing
driver->partSize = 16;
// Coverage for noop functions
// -----------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(storagePathSyncP(s3, strNew("path")), "path sync is a noop");
@ -383,10 +441,145 @@ testRun(void)
TEST_RESULT_STR_Z(strNewBuf(storageGetP(storageNewReadP(s3, strNew("file0.txt")))), "", "get zero-length file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("switch to temp credentials");
hrnServerScriptClose(service);
argList = strLstDup(commonArgList);
strLstAdd(argList, strNewFmt("--" CFGOPT_REPO1_S3_HOST "=%s:%u", strZ(host), port));
strLstAdd(argList, strNewFmt("--" CFGOPT_REPO1_S3_ROLE "=%s", strZ(credRole)));
strLstAddZ(argList, "--" CFGOPT_REPO1_S3_KEY_TYPE "=" STORAGE_S3_KEY_TYPE_AUTO);
harnessCfgLoad(cfgCmdArchivePush, argList);
s3 = storageRepoGet(STORAGE_S3_TYPE_STR, true);
driver = (StorageS3 *)s3->driver;
TEST_RESULT_STR(s3->path, path, "check path");
TEST_RESULT_STR(driver->credRole, credRole, "check role");
TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
TEST_RESULT_BOOL(storageFeature(s3, storageFeatureCompress), false, "check compress feature");
// Set partSize to a small value for testing
driver->partSize = 16;
// Testing requires the auth http client to be redirected
driver->credHost = hrnServerHost();
driver->credHttpClient = httpClientNew(sckClientNew(host, authPort, 5000), 5000);
// Now that we have checked the role when set explicitly, null it out to make sure it is retrieved automatically
driver->credRole = NULL;
hrnServerScriptAccept(service);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error when retrieving role");
hrnServerScriptAccept(auth);
testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_URI);
testResponseP(auth, .http = "1.0", .code = 301);
hrnServerScriptClose(auth);
TEST_ERROR_FMT(
storageGetP(storageNewReadP(s3, strNew("file.txt"))), ProtocolError,
"HTTP request failed with 301:\n"
"*** URI/Query ***:\n"
"/latest/meta-data/iam/security-credentials\n"
"*** Request Headers ***:\n"
"content-length: 0\n"
"host: %s",
strZ(hrnServerHost()));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("missing role");
hrnServerScriptAccept(auth);
testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_URI);
testResponseP(auth, .http = "1.0", .code = 404);
hrnServerScriptClose(auth);
TEST_ERROR(
storageGetP(storageNewReadP(s3, strNew("file.txt"))), ProtocolError,
"role to retrieve temporary credentials not found\n"
"HINT: is a valid IAM role associated with this instance?");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error when retrieving temp credentials");
hrnServerScriptAccept(auth);
testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_URI);
testResponseP(auth, .http = "1.0", .content = strZ(credRole));
hrnServerScriptClose(auth);
hrnServerScriptAccept(auth);
testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
testResponseP(auth, .http = "1.0", .code = 300);
hrnServerScriptClose(auth);
TEST_ERROR_FMT(
storageGetP(storageNewReadP(s3, strNew("file.txt"))), ProtocolError,
"HTTP request failed with 300:\n"
"*** URI/Query ***:\n"
"/latest/meta-data/iam/security-credentials/credrole\n"
"*** Request Headers ***:\n"
"content-length: 0\n"
"host: %s",
strZ(hrnServerHost()));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("invalid temp credentials role");
hrnServerScriptAccept(auth);
testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
testResponseP(auth, .http = "1.0", .code = 404);
hrnServerScriptClose(auth);
TEST_ERROR_FMT(
storageGetP(storageNewReadP(s3, strNew("file.txt"))), ProtocolError,
"role '%s' not found\n"
"HINT: is '%s' a valid IAM role associated with this instance?",
strZ(credRole), strZ(credRole));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("invalid code when retrieving temp credentials");
hrnServerScriptAccept(auth);
testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
testResponseP(auth, .http = "1.0", .content = "{\"Code\":\"IAM role is not configured\"}");
hrnServerScriptClose(auth);
TEST_ERROR(
storageGetP(storageNewReadP(s3, strNew("file.txt"))), FormatError,
"unable to retrieve temporary credentials: IAM role is not configured");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("non-404 error");
testRequestP(service, s3, HTTP_VERB_GET, "/file.txt");
hrnServerScriptAccept(auth);
testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
testResponseP(
auth,
.content = strZ(
strNewFmt(
"{\"Code\":\"Success\",\"AccessKeyId\":\"x\",\"SecretAccessKey\":\"y\",\"Token\":\"z\""
",\"Expiration\":\"%s\"}",
strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC - 1))))));
hrnServerScriptClose(auth);
testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .accessKey = "x", .securityToken = "z");
testResponseP(service, .code = 303, .content = "CONTENT");
StorageRead *read = NULL;
@ -411,12 +604,33 @@ testRun(void)
"*** Response Content ***:\n"
"CONTENT")
// Check that temp credentials were set
TEST_RESULT_STR_Z(driver->accessKey, "x", "check access key");
TEST_RESULT_STR_Z(driver->secretAccessKey, "y", "check secret access key");
TEST_RESULT_STR_Z(driver->securityToken, "z", "check security token");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in one part");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD");
hrnServerScriptAccept(auth);
testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_URI "/%s", strZ(credRole))));
testResponseP(
auth,
.content = strZ(
strNewFmt(
"{\"Code\":\"Success\",\"AccessKeyId\":\"xx\",\"SecretAccessKey\":\"yy\",\"Token\":\"zz\""
",\"Expiration\":\"%s\"}",
strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC * 2))))));
hrnServerScriptClose(auth);
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz");
testResponseP(service);
// Make a copy of the signing key to verify that it gets changed when the keys are updated
const Buffer *oldSigningKey = bufDup(driver->signingKey);
StorageWrite *write = NULL;
TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write");
TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("ABCD")), "write");
@ -431,6 +645,17 @@ testRun(void)
TEST_RESULT_VOID(storageWriteS3Close(write->driver), "close file again");
// Check that temp credentials were changed
TEST_RESULT_STR_Z(driver->accessKey, "xx", "check access key");
TEST_RESULT_STR_Z(driver->secretAccessKey, "yy", "check secret access key");
TEST_RESULT_STR_Z(driver->securityToken, "zz", "check security token");
// Check that the signing key changed
TEST_RESULT_BOOL(bufEq(driver->signingKey, oldSigningKey), false, "signing key changed");
// Auth service no longer needed
hrnServerScriptEnd(auth);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write zero-length file");