db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-05 00:28:52 +02:00
Code Issues Releases Activity

Allow path-style URIs in S3 driver.

Browse Source 
Although path-style URIs have been deprecated by AWS, they may still be used with products like Minio because no additional DNS configuration is required.

Path-style URIs must be explicitly enabled since it is not clear how they can be auto-detected reliably.  More importantly, faulty detection could cause regressions in current installations.
This commit is contained in:
David Steele
2020-01-12 11:31:06 -07:00
parent 069345d339
commit fe263e87b1
14 changed files with 252 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
build/lib/pgBackRestBuild/Config/Data.pm
View File

@ -283,6 +283,8 @@ use constant CFGOPT_REPO_S3_REGION => CFGDEF_RE
push @EXPORT, qw(CFGOPT_REPO_S3_REGION);
use constant CFGOPT_REPO_S3_TOKEN => CFGDEF_REPO_S3 . '-token';
push @EXPORT, qw(CFGOPT_REPO_S3_TOKEN);
use constant CFGOPT_REPO_S3_URI_STYLE => CFGDEF_REPO_S3 . '-uri-style';
push @EXPORT, qw(CFGOPT_REPO_S3_URI_STYLE);
use constant CFGOPT_REPO_S3_VERIFY_TLS => CFGDEF_REPO_S3 . '-verify-tls';
push @EXPORT, qw(CFGOPT_REPO_S3_VERIFY_TLS);
@ -401,6 +403,13 @@ use constant CFGOPTVAL_REPO_CIPHER_TYPE_NONE => 'none';
use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc';
push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
# Repo S3 URI style
#-----------------------------------------------------------------------------------------------------------------------------------
use constant CFGOPTVAL_REPO_S3_URI_STYLE_HOST => 'host';
push @EXPORT, qw(CFGOPTVAL_REPO_S3_URI_STYLE_HOST);
use constant CFGOPTVAL_REPO_S3_URI_STYLE_PATH => 'path';
push @EXPORT, qw(CFGOPTVAL_REPO_S3_URI_STYLE_PATH);
# Info output
#-----------------------------------------------------------------------------------------------------------------------------------
use constant CFGOPTVAL_OUTPUT_TEXT => 'text';
@ -1839,6 +1848,22 @@ my %hConfigDefine =
&CFGDEF_COMMAND => CFGOPT_REPO_TYPE,
},
&CFGOPT_REPO_S3_URI_STYLE =>
{
&CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL,
&CFGDEF_TYPE => CFGDEF_TYPE_STRING,
&CFGDEF_PREFIX => CFGDEF_PREFIX_REPO,
&CFGDEF_INDEX_TOTAL => CFGDEF_INDEX_REPO,
&CFGDEF_DEFAULT => CFGOPTVAL_REPO_S3_URI_STYLE_HOST,
&CFGDEF_ALLOW_LIST =>
[
&CFGOPTVAL_REPO_S3_URI_STYLE_HOST,
&CFGOPTVAL_REPO_S3_URI_STYLE_PATH,
],
&CFGDEF_COMMAND => CFGOPT_REPO_TYPE,
&CFGDEF_DEPEND => CFGOPT_REPO_S3_BUCKET,
},
&CFGOPT_REPO_S3_VERIFY_TLS =>
{
&CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL,

13
doc/xml/reference.xml
View File

@ -455,6 +455,19 @@
<example>us-east-1</example>
</config-key>
<!-- CONFIG - REPO SECTION - REPO-S3-URI-STYLE KEY -->
<config-key id="repo-s3-uri-style" name="S3 Repository URI Style">
<summary>S3 URI Style.</summary>
<text>The following URI styles are supported:
<ul>
<li><id>host</id> - Connect to <id>bucket.endpoint</id> host.</li>
<li><id>path</id> - Connect to <id>endpoint</id> host and prepend bucket to URIs.</li>
</ul></text>
<example>path</example>
</config-key>
<!-- CONFIG - REPO SECTION - REPO-S3-VERIFY-TLS KEY -->
<config-key id="repo-s3-verify-tls" name="S3 Repository Verify TLS">
<summary>Verify S3 server certificate.</summary>

4
doc/xml/release.xml
View File

@ -36,6 +36,10 @@
<p>Specifies the database user name when connecting to <postgres/>. If not specified <backrest/> will connect with the local OS user or <id>PGUSER</id>, which was the previous behavior.</p>
</release-item>
<release-item>
<p>Allow path-style URIs in <proper>S3</proper> driver.</p>
</release-item>
</release-feature-list>
<release-improvement-list>

6
lib/pgBackRest/LibCAuto.pm
View File

@ -27,6 +27,9 @@ sub libcAutoConstant
CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_DIFF => 'diff',
CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_INCR => 'incr',
CFGOPTVAL_REPO_S3_URI_STYLE_HOST => 'host',
CFGOPTVAL_REPO_S3_URI_STYLE_PATH => 'path',
CFGOPTVAL_REPO_TYPE_CIFS => 'cifs',
CFGOPTVAL_REPO_TYPE_POSIX => 'posix',
CFGOPTVAL_REPO_TYPE_S3 => 's3',
@ -91,6 +94,8 @@ sub libcAutoExportTag
'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_FULL',
'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_DIFF',
'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_INCR',
'CFGOPTVAL_REPO_S3_URI_STYLE_HOST',
'CFGOPTVAL_REPO_S3_URI_STYLE_PATH',
'CFGOPTVAL_REPO_TYPE_CIFS',
'CFGOPTVAL_REPO_TYPE_POSIX',
'CFGOPTVAL_REPO_TYPE_S3',
@ -286,6 +291,7 @@ sub libcAutoExportTag
'CFGOPT_REPO_S3_PORT',
'CFGOPT_REPO_S3_REGION',
'CFGOPT_REPO_S3_TOKEN',
'CFGOPT_REPO_S3_URI_STYLE',
'CFGOPT_REPO_S3_VERIFY_TLS',
'CFGOPT_REPO_TYPE',
'CFGOPT_RESUME',

9
src/config/config.auto.c
View File

@ -458,6 +458,7 @@ STRING_EXTERN(CFGOPT_REPO1_S3_KEY_SECRET_STR, CFGOPT_REPO1
STRING_EXTERN(CFGOPT_REPO1_S3_PORT_STR, CFGOPT_REPO1_S3_PORT);
STRING_EXTERN(CFGOPT_REPO1_S3_REGION_STR, CFGOPT_REPO1_S3_REGION);
STRING_EXTERN(CFGOPT_REPO1_S3_TOKEN_STR, CFGOPT_REPO1_S3_TOKEN);
STRING_EXTERN(CFGOPT_REPO1_S3_URI_STYLE_STR, CFGOPT_REPO1_S3_URI_STYLE);
STRING_EXTERN(CFGOPT_REPO1_S3_VERIFY_TLS_STR, CFGOPT_REPO1_S3_VERIFY_TLS);
STRING_EXTERN(CFGOPT_REPO1_TYPE_STR, CFGOPT_REPO1_TYPE);
STRING_EXTERN(CFGOPT_RESUME_STR, CFGOPT_RESUME);
@ -1728,6 +1729,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3Token)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
CONFIG_OPTION_NAME(CFGOPT_REPO1_S3_URI_STYLE)
CONFIG_OPTION_INDEX(0)
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3UriStyle)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(

5
src/config/config.auto.h
View File

@ -365,6 +365,8 @@ Option constants
STRING_DECLARE(CFGOPT_REPO1_S3_REGION_STR);
#define CFGOPT_REPO1_S3_TOKEN "repo1-s3-token"
STRING_DECLARE(CFGOPT_REPO1_S3_TOKEN_STR);
#define CFGOPT_REPO1_S3_URI_STYLE "repo1-s3-uri-style"
STRING_DECLARE(CFGOPT_REPO1_S3_URI_STYLE_STR);
#define CFGOPT_REPO1_S3_VERIFY_TLS "repo1-s3-verify-tls"
STRING_DECLARE(CFGOPT_REPO1_S3_VERIFY_TLS_STR);
#define CFGOPT_REPO1_TYPE "repo1-type"
@ -398,7 +400,7 @@ Option constants
#define CFGOPT_TYPE "type"
STRING_DECLARE(CFGOPT_TYPE_STR);
#define CFG_OPTION_TOTAL 172
#define CFG_OPTION_TOTAL 173
/***********************************************************************************************************************************
Command enum
@ -588,6 +590,7 @@ typedef enum
cfgOptRepoS3Port,
cfgOptRepoS3Region,
cfgOptRepoS3Token,
cfgOptRepoS3UriStyle,
cfgOptRepoS3VerifyTls,
cfgOptRepoType,
cfgOptResume,

62
src/config/define.auto.c
View File

@ -3938,6 +3938,68 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
CFGDEFDATA_OPTION_NAME("repo-s3-uri-style")
CFGDEFDATA_OPTION_REQUIRED(true)
CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString)
CFGDEFDATA_OPTION_INTERNAL(false)
CFGDEFDATA_OPTION_INDEX_TOTAL(1)
CFGDEFDATA_OPTION_SECURE(false)
CFGDEFDATA_OPTION_HELP_SECTION("repository")
CFGDEFDATA_OPTION_HELP_SUMMARY("S3 URI Style.")
CFGDEFDATA_OPTION_HELP_DESCRIPTION
(
"The following URI styles are supported:\n"
"\n"
"* host - Connect to bucket.endpoint host.\n"
"* path - Connect to endpoint host and prepend bucket to URIs."
)
CFGDEFDATA_OPTION_COMMAND_LIST
(
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePushAsync)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdLocal)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdLs)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRemote)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStart)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStop)
)
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
(
"host",
"path"
)
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
cfgDefOptRepoType,
"s3"
)
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("host")
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(

1
src/config/define.auto.h
View File

@ -131,6 +131,7 @@ typedef enum
cfgDefOptRepoS3Port,
cfgDefOptRepoS3Region,
cfgDefOptRepoS3Token,
cfgDefOptRepoS3UriStyle,
cfgDefOptRepoS3VerifyTls,
cfgDefOptRepoType,
cfgDefOptResume,

13
src/config/parse.auto.c
View File

@ -2120,6 +2120,18 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3Token,
},
// repo-s3-uri-style option
// -----------------------------------------------------------------------------------------------------------------------------
{
.name = CFGOPT_REPO1_S3_URI_STYLE,
.has_arg = required_argument,
.val = PARSE_OPTION_FLAG | cfgOptRepoS3UriStyle,
},
{
.name = "reset-" CFGOPT_REPO1_S3_URI_STYLE,
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3UriStyle,
},
// repo-s3-verify-tls option and deprecations
// -----------------------------------------------------------------------------------------------------------------------------
{
@ -2489,6 +2501,7 @@ static const ConfigOption optionResolveOrder[] =
cfgOptRepoS3Port,
cfgOptRepoS3Region,
cfgOptRepoS3Token,
cfgOptRepoS3UriStyle,
cfgOptRepoS3VerifyTls,
cfgOptTarget,
cfgOptTargetAction,

1
src/storage/helper.c
View File

@ -369,6 +369,7 @@ storageRepoGet(const String *type, bool write)
result = storageS3New(
cfgOptionStr(cfgOptRepoPath), write, storageRepoPathExpression, cfgOptionStr(cfgOptRepoS3Bucket), endPoint,
strEqZ(cfgOptionStr(cfgOptRepoS3UriStyle), STORAGE_S3_URI_STYLE_HOST) ? storageS3UriStyleHost : storageS3UriStylePath,
cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key), cfgOptionStr(cfgOptRepoS3KeySecret),
cfgOptionTest(cfgOptRepoS3Token) ? cfgOptionStr(cfgOptRepoS3Token) : NULL, STORAGE_S3_PARTSIZE_MIN,
STORAGE_S3_DELETE_MAX, host, port, STORAGE_S3_TIMEOUT_DEFAULT, cfgOptionBool(cfgOptRepoS3VerifyTls),

16
src/storage/s3/storage.c
View File

@ -98,6 +98,7 @@ struct StorageS3
const String *securityToken; // Security token, if any
size_t partSize; // Part size for multi-part upload
unsigned int deleteMax; // Maximum objects that can be deleted in one request
StorageS3UriStyle uriStyle; // Path or host style URIs
const String *bucketEndpoint; // Set to {bucket}.{endpoint}
unsigned int port; // Host port
@ -258,6 +259,10 @@ storageS3Request(
unsigned int retryRemaining = 2;
bool done;
// When using path-style URIs the bucket name needs to be prepended
if (this->uriStyle == storageS3UriStylePath)
uri = strNewFmt("/%s%s", strPtr(this->bucket), strPtr(uri));
do
{
done = true;
@ -917,9 +922,9 @@ static const StorageInterface storageInterfaceS3 =
Storage *
storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey,
const String *securityToken, size_t partSize, unsigned int deleteMax, const String *host, unsigned int port, TimeMSec timeout,
bool verifyPeer, const String *caFile, const String *caPath)
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, const String *accessKey,
const String *secretAccessKey, const String *securityToken, size_t partSize, unsigned int deleteMax, const String *host,
unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@ -927,6 +932,7 @@ storageS3New(
FUNCTION_LOG_PARAM(FUNCTIONP, pathExpressionFunction);
FUNCTION_LOG_PARAM(STRING, bucket);
FUNCTION_LOG_PARAM(STRING, endPoint);
FUNCTION_LOG_PARAM(ENUM, uriStyle);
FUNCTION_LOG_PARAM(STRING, region);
FUNCTION_TEST_PARAM(STRING, accessKey);
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
@ -962,7 +968,9 @@ storageS3New(
driver->securityToken = strDup(securityToken);
driver->partSize = partSize;
driver->deleteMax = deleteMax;
driver->bucketEndpoint = strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint));
driver->uriStyle = uriStyle;
driver->bucketEndpoint = uriStyle == storageS3UriStyleHost ?
strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint)) : strDup(endPoint);
driver->port = port;
// Force the signing key to be generated on the first run

18
src/storage/s3/storage.h
View File

@ -12,6 +12,18 @@ Storage type
#define STORAGE_S3_TYPE "s3"
STRING_DECLARE(STORAGE_S3_TYPE_STR);
/***********************************************************************************************************************************
URI style
***********************************************************************************************************************************/
typedef enum
{
storageS3UriStyleHost,
storageS3UriStylePath,
} StorageS3UriStyle;
#define STORAGE_S3_URI_STYLE_HOST "host"
#define STORAGE_S3_URI_STYLE_PATH "path"
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
@ -24,8 +36,8 @@ Constructor
***********************************************************************************************************************************/
Storage *storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey,
const String *securityToken, size_t partSize, unsigned int deleteMax, const String *host, unsigned int port, TimeMSec timeout,
bool verifyPeer, const String *caFile, const String *caPath);
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, const String *accessKey,
const String *secretAccessKey, const String *securityToken, size_t partSize, unsigned int deleteMax, const String *host,
unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
#endif

1
test/src/module/command/helpTest.c
View File

@ -210,6 +210,7 @@ testRun(void)
" --repo-s3-port s3 repository port [default=443]\n"
" --repo-s3-region s3 repository region\n"
" --repo-s3-token s3 repository security token\n"
" --repo-s3-uri-style s3 URI Style [default=host]\n"
" --repo-s3-verify-tls verify S3 server certificate [default=y]\n"
" --repo-type type of storage used for the repository\n"
" [default=posix]\n"

144
test/src/module/storage/s3Test.c
View File

@ -9,14 +9,14 @@ Test S3 Storage
/***********************************************************************************************************************************
Test server
***********************************************************************************************************************************/
#define S3_TEST_HOST "bucket.s3.amazonaws.com"
#define S3_TEST_HOST "s3.amazonaws.com"
#define DATE_REPLACE "????????"
#define DATETIME_REPLACE "????????T??????Z"
#define SHA256_REPLACE \
"????????????????????????????????????????????????????????????????"
static const char *
testS3ServerRequest(const char *verb, const char *uri, const char *content)
testS3ServerRequest(const char *verb, const char *uri, const char *content, StorageS3UriStyle uriStyle)
{
String *request = strNewFmt(
"%s %s HTTP/1.1\r\n"
@ -40,9 +40,13 @@ testS3ServerRequest(const char *verb, const char *uri, const char *content)
strCatFmt(request, "content-md5:%s\r\n", md5Hash);
}
if (uriStyle == storageS3UriStyleHost)
strCatFmt(request, "host:bucket." S3_TEST_HOST "\r\n");
else
strCatFmt(request, "host:" S3_TEST_HOST "\r\n");
strCatFmt(
request,
"host:" S3_TEST_HOST "\r\n"
"x-amz-content-sha256:%s\r\n"
"x-amz-date:" DATETIME_REPLACE "\r\n"
"\r\n",
@ -88,29 +92,29 @@ testS3Server(void)
// storageS3NewRead() and StorageS3FileRead
// -------------------------------------------------------------------------------------------------------------------------
// Ignore missing file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/fi%26le.txt", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/fi%26le.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL));
// Error on missing file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL));
// Get file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, "this is a sample file"));
// Get zero-length file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file0.txt", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file0.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// Throw non-404 error
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(303, "Some bad status", NULL, "CONTENT"));
// storageS3NewWrite() and StorageWriteS3
// -------------------------------------------------------------------------------------------------------------------------
// File is written all at once
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD", storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
403, "Forbidden", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@ -125,15 +129,15 @@ testS3Server(void)
"</Error>"));
harnessTlsServerAccept();
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD", storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// Zero-length file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", ""));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "", storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// File is written in chunks with nothing left over on close
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@ -143,9 +147,11 @@ testS3Server(void)
"<UploadId>WxRt</UploadId>"
"</InitiateMultipartUploadResult>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", "1234567890123456"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", "1234567890123456", storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "etag:WxRt1", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", "7890123456789012"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", "7890123456789012", storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "eTag:WxRt2", NULL));
harnessTlsServerExpect(testS3ServerRequest(
@ -154,11 +160,12 @@ testS3Server(void)
"<CompleteMultipartUpload>"
"<Part><PartNumber>1</PartNumber><ETag>WxRt1</ETag></Part>"
"<Part><PartNumber>2</PartNumber><ETag>WxRt2</ETag></Part>"
"</CompleteMultipartUpload>\n"));
"</CompleteMultipartUpload>\n",
storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// File is written in chunks with something left over on close
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@ -168,9 +175,11 @@ testS3Server(void)
"<UploadId>RR55</UploadId>"
"</InitiateMultipartUploadResult>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", "1234567890123456"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", "1234567890123456", storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "etag:RR551", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", "7890"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", "7890", storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "eTag:RR552", NULL));
harnessTlsServerExpect(testS3ServerRequest(
@ -179,27 +188,28 @@ testS3Server(void)
"<CompleteMultipartUpload>"
"<Part><PartNumber>1</PartNumber><ETag>RR551</ETag></Part>"
"<Part><PartNumber>2</PartNumber><ETag>RR552</ETag></Part>"
"</CompleteMultipartUpload>\n"));
"</CompleteMultipartUpload>\n",
storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// storageDriverExists()
// -------------------------------------------------------------------------------------------------------------------------
// File missing
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL));
// File exists
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:999", NULL));
// Info()
// -------------------------------------------------------------------------------------------------------------------------
// File missing
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL));
// File exists
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
200, "OK",
"content-length:9999\r\n"
@ -209,7 +219,7 @@ testS3Server(void)
// InfoList()
// -------------------------------------------------------------------------------------------------------------------------
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL));
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStyleHost));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -228,10 +238,10 @@ testS3Server(void)
// storageDriverList()
// -------------------------------------------------------------------------------------------------------------------------
// Throw errors
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse( 344, "Another bad status", NULL, NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
344, "Another bad status with xml", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@ -239,7 +249,7 @@ testS3Server(void)
"<Code>SomeOtherCode</Code>"
"</Error>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
403, "Forbidden", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@ -247,7 +257,7 @@ testS3Server(void)
"<Code>RequestTimeTooSkewed</Code>"
"<Message>The difference between the request time and the current time is too large.</Message>"
"</Error>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
403, "Forbidden", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@ -255,7 +265,7 @@ testS3Server(void)
"<Code>RequestTimeTooSkewed</Code>"
"<Message>The difference between the request time and the current time is too large.</Message>"
"</Error>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
403, "Forbidden", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@ -265,7 +275,7 @@ testS3Server(void)
"</Error>"));
// list a file/path in root
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -280,7 +290,8 @@ testS3Server(void)
"</ListBucketResult>"));
// list a file in root with expression
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test", NULL));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test", NULL, storageS3UriStyleHost));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -293,7 +304,7 @@ testS3Server(void)
// list files with continuation
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL));
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStyleHost));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -316,7 +327,7 @@ testS3Server(void)
HTTP_VERB_GET,
"/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2"
"&prefix=path%2Fto%2F",
NULL));
NULL, storageS3UriStyleHost));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -332,7 +343,7 @@ testS3Server(void)
// list files with expression
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest", NULL));
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest", NULL, storageS3UriStyleHost));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -357,8 +368,13 @@ testS3Server(void)
// storageDriverPathRemove()
// -------------------------------------------------------------------------------------------------------------------------
// Switch to path-style URIs
harnessTlsServerClose();
harnessTlsServerAccept();
// delete files from root
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2", NULL));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2", NULL, storageS3UriStylePath));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -373,18 +389,20 @@ testS3Server(void)
"</ListBucketResult>"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_POST, "/?delete=",
testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
"<Object><Key>test1.txt</Key></Object>"
"<Object><Key>path1/xxx.zzz</Key></Object>"
"</Delete>\n"));
"</Delete>\n",
storageS3UriStylePath));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL, "<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"></DeleteResult>"));
// nothing to do in empty subpath
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2F", NULL));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F", NULL, storageS3UriStylePath));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -394,7 +412,7 @@ testS3Server(void)
// delete with continuation
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2Fto%2F", NULL));
testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStylePath));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -410,7 +428,9 @@ testS3Server(void)
"</ListBucketResult>"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F", NULL));
testS3ServerRequest(
HTTP_VERB_GET, "/bucket/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F", NULL,
storageS3UriStylePath));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -425,25 +445,27 @@ testS3Server(void)
"</ListBucketResult>"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_POST, "/?delete=",
testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
"<Object><Key>path/to/test1.txt</Key></Object>"
"<Object><Key>path/to/test3.txt</Key></Object>"
"</Delete>\n"));
"</Delete>\n",
storageS3UriStylePath));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_POST, "/?delete=",
testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
"<Object><Key>path/to/test2.txt</Key></Object>"
"</Delete>\n"));
"</Delete>\n",
storageS3UriStylePath));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// delete error
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2F", NULL));
testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F", NULL, storageS3UriStylePath));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -458,12 +480,13 @@ testS3Server(void)
"</ListBucketResult>"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_POST, "/?delete=",
testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
"<Object><Key>path/sample.txt</Key></Object>"
"<Object><Key>path/sample2.txt</Key></Object>"
"</Delete>\n"));
"</Delete>\n",
storageS3UriStylePath));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK", NULL,
@ -475,7 +498,7 @@ testS3Server(void)
// storageDriverRemove()
// -------------------------------------------------------------------------------------------------------------------------
// remove file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_DELETE, "/path/to/test.txt", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_DELETE, "/bucket/path/to/test.txt", NULL, storageS3UriStylePath));
harnessTlsServerReply(testS3ServerResponse(204, "No Content", NULL, NULL));
harnessTlsServerClose();
@ -669,8 +692,8 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
StorageS3 *driver = (StorageS3 *)storageDriver(
storageS3New(
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, 16, 2, NULL, 0, 0, testContainer(),
NULL, NULL));
path, true, NULL, bucket, endPoint, storageS3UriStyleHost, region, accessKey, secretAccessKey, NULL, 16, 2, NULL, 0,
0, testContainer(), NULL, NULL));
HttpHeader *header = httpHeaderNew(NULL);
@ -717,8 +740,8 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
driver = (StorageS3 *)storageDriver(
storageS3New(
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, securityToken, 16, 2, NULL, 0, 0,
testContainer(), NULL, NULL));
path, true, NULL, bucket, endPoint, storageS3UriStyleHost, region, accessKey, secretAccessKey, securityToken, 16, 2,
NULL, 0, 0, testContainer(), NULL, NULL));
TEST_RESULT_VOID(
storageS3Auth(driver, strNew("GET"), strNew("/"), query, strNew("20170606T121212Z"), header, HASH_TYPE_SHA256_ZERO_STR),
@ -737,8 +760,8 @@ testRun(void)
testS3Server();
Storage *s3 = storageS3New(
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, 16, 2, host, port, 1000, testContainer(),
NULL, NULL);
path, true, NULL, bucket, endPoint, storageS3UriStyleHost, region, accessKey, secretAccessKey, NULL, 16, 2, host, port,
1000, testContainer(), NULL, NULL);
// Coverage for noop functions
// -------------------------------------------------------------------------------------------------------------------------
@ -767,7 +790,7 @@ testRun(void)
"*** Request Headers ***:\n"
"authorization: <redacted>\n"
"content-length: 0\n"
"host: " S3_TEST_HOST "\n"
"host: bucket." S3_TEST_HOST "\n"
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date: <redacted>\n"
"*** Response Headers ***:\n"
@ -854,7 +877,7 @@ testRun(void)
"*** Request Headers ***:\n"
"authorization: <redacted>\n"
"content-length: 0\n"
"host: " S3_TEST_HOST "\n"
"host: bucket." S3_TEST_HOST "\n"
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date: <redacted>");
TEST_ERROR(storageListP(s3, strNew("/")), ProtocolError,
@ -864,7 +887,7 @@ testRun(void)
"*** Request Headers ***:\n"
"authorization: <redacted>\n"
"content-length: 0\n"
"host: " S3_TEST_HOST "\n"
"host: bucket." S3_TEST_HOST "\n"
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date: <redacted>\n"
"*** Response Headers ***:\n"
@ -878,7 +901,7 @@ testRun(void)
"*** Request Headers ***:\n"
"authorization: <redacted>\n"
"content-length: 0\n"
"host: " S3_TEST_HOST "\n"
"host: bucket." S3_TEST_HOST "\n"
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date: <redacted>\n"
"*** Response Headers ***:\n"
@ -900,6 +923,11 @@ testRun(void)
// storageDriverPathRemove()
// -------------------------------------------------------------------------------------------------------------------------
// Switch to path-style URIs
s3 = storageS3New(
path, true, NULL, bucket, endPoint, storageS3UriStylePath, region, accessKey, secretAccessKey, NULL, 16, 2, host, port,
1000, testContainer(), NULL, NULL);
TEST_ERROR(
storagePathRemoveP(s3, strNew("/")), AssertError,
"assertion 'param.recurse || storageFeature(this, storageFeaturePath)' failed");