diff --git a/build/lib/pgBackRestBuild/Config/Data.pm b/build/lib/pgBackRestBuild/Config/Data.pm index 0a0203478..f954d29af 100644 --- a/build/lib/pgBackRestBuild/Config/Data.pm +++ b/build/lib/pgBackRestBuild/Config/Data.pm @@ -283,6 +283,8 @@ use constant CFGOPT_REPO_S3_REGION => CFGDEF_RE push @EXPORT, qw(CFGOPT_REPO_S3_REGION); use constant CFGOPT_REPO_S3_TOKEN => CFGDEF_REPO_S3 . '-token'; push @EXPORT, qw(CFGOPT_REPO_S3_TOKEN); +use constant CFGOPT_REPO_S3_URI_STYLE => CFGDEF_REPO_S3 . '-uri-style'; + push @EXPORT, qw(CFGOPT_REPO_S3_URI_STYLE); use constant CFGOPT_REPO_S3_VERIFY_TLS => CFGDEF_REPO_S3 . '-verify-tls'; push @EXPORT, qw(CFGOPT_REPO_S3_VERIFY_TLS); @@ -401,6 +403,13 @@ use constant CFGOPTVAL_REPO_CIPHER_TYPE_NONE => 'none'; use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc'; push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC); +# Repo S3 URI style +#----------------------------------------------------------------------------------------------------------------------------------- +use constant CFGOPTVAL_REPO_S3_URI_STYLE_HOST => 'host'; + push @EXPORT, qw(CFGOPTVAL_REPO_S3_URI_STYLE_HOST); +use constant CFGOPTVAL_REPO_S3_URI_STYLE_PATH => 'path'; + push @EXPORT, qw(CFGOPTVAL_REPO_S3_URI_STYLE_PATH); + # Info output #----------------------------------------------------------------------------------------------------------------------------------- use constant CFGOPTVAL_OUTPUT_TEXT => 'text'; @@ -1839,6 +1848,22 @@ my %hConfigDefine = &CFGDEF_COMMAND => CFGOPT_REPO_TYPE, }, + &CFGOPT_REPO_S3_URI_STYLE => + { + &CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL, + &CFGDEF_TYPE => CFGDEF_TYPE_STRING, + &CFGDEF_PREFIX => CFGDEF_PREFIX_REPO, + &CFGDEF_INDEX_TOTAL => CFGDEF_INDEX_REPO, + &CFGDEF_DEFAULT => CFGOPTVAL_REPO_S3_URI_STYLE_HOST, + &CFGDEF_ALLOW_LIST => + [ + &CFGOPTVAL_REPO_S3_URI_STYLE_HOST, + &CFGOPTVAL_REPO_S3_URI_STYLE_PATH, + ], + &CFGDEF_COMMAND => CFGOPT_REPO_TYPE, + &CFGDEF_DEPEND => CFGOPT_REPO_S3_BUCKET, + }, + &CFGOPT_REPO_S3_VERIFY_TLS => { &CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL, diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index aaae132ac..714c7d31e 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -455,6 +455,19 @@ us-east-1 + + + S3 URI Style. + + The following URI styles are supported: + + + path + + Verify S3 server certificate. diff --git a/doc/xml/release.xml b/doc/xml/release.xml index a4bd72ecd..619cf468b 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -36,6 +36,10 @@

Specifies the database user name when connecting to . If not specified will connect with the local OS user or PGUSER, which was the previous behavior.

+ + +

Allow path-style URIs in S3 driver.

+ diff --git a/lib/pgBackRest/LibCAuto.pm b/lib/pgBackRest/LibCAuto.pm index 3340df9c6..b2a30b219 100644 --- a/lib/pgBackRest/LibCAuto.pm +++ b/lib/pgBackRest/LibCAuto.pm @@ -27,6 +27,9 @@ sub libcAutoConstant CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_DIFF => 'diff', CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_INCR => 'incr', + CFGOPTVAL_REPO_S3_URI_STYLE_HOST => 'host', + CFGOPTVAL_REPO_S3_URI_STYLE_PATH => 'path', + CFGOPTVAL_REPO_TYPE_CIFS => 'cifs', CFGOPTVAL_REPO_TYPE_POSIX => 'posix', CFGOPTVAL_REPO_TYPE_S3 => 's3', @@ -91,6 +94,8 @@ sub libcAutoExportTag 'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_FULL', 'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_DIFF', 'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_INCR', + 'CFGOPTVAL_REPO_S3_URI_STYLE_HOST', + 'CFGOPTVAL_REPO_S3_URI_STYLE_PATH', 'CFGOPTVAL_REPO_TYPE_CIFS', 'CFGOPTVAL_REPO_TYPE_POSIX', 'CFGOPTVAL_REPO_TYPE_S3', @@ -286,6 +291,7 @@ sub libcAutoExportTag 'CFGOPT_REPO_S3_PORT', 'CFGOPT_REPO_S3_REGION', 'CFGOPT_REPO_S3_TOKEN', + 'CFGOPT_REPO_S3_URI_STYLE', 'CFGOPT_REPO_S3_VERIFY_TLS', 'CFGOPT_REPO_TYPE', 'CFGOPT_RESUME', diff --git a/src/config/config.auto.c b/src/config/config.auto.c index ed6a06d22..71994b9b6 100644 --- a/src/config/config.auto.c +++ b/src/config/config.auto.c @@ -458,6 +458,7 @@ STRING_EXTERN(CFGOPT_REPO1_S3_KEY_SECRET_STR, CFGOPT_REPO1 STRING_EXTERN(CFGOPT_REPO1_S3_PORT_STR, CFGOPT_REPO1_S3_PORT); STRING_EXTERN(CFGOPT_REPO1_S3_REGION_STR, CFGOPT_REPO1_S3_REGION); STRING_EXTERN(CFGOPT_REPO1_S3_TOKEN_STR, CFGOPT_REPO1_S3_TOKEN); +STRING_EXTERN(CFGOPT_REPO1_S3_URI_STYLE_STR, CFGOPT_REPO1_S3_URI_STYLE); STRING_EXTERN(CFGOPT_REPO1_S3_VERIFY_TLS_STR, CFGOPT_REPO1_S3_VERIFY_TLS); STRING_EXTERN(CFGOPT_REPO1_TYPE_STR, CFGOPT_REPO1_TYPE); STRING_EXTERN(CFGOPT_RESUME_STR, CFGOPT_RESUME); @@ -1728,6 +1729,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3Token) ) + //------------------------------------------------------------------------------------------------------------------------------ + CONFIG_OPTION + ( + CONFIG_OPTION_NAME(CFGOPT_REPO1_S3_URI_STYLE) + CONFIG_OPTION_INDEX(0) + CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3UriStyle) + ) + //------------------------------------------------------------------------------------------------------------------------------ CONFIG_OPTION ( diff --git a/src/config/config.auto.h b/src/config/config.auto.h index 3b0941590..bf66c59e3 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -365,6 +365,8 @@ Option constants STRING_DECLARE(CFGOPT_REPO1_S3_REGION_STR); #define CFGOPT_REPO1_S3_TOKEN "repo1-s3-token" STRING_DECLARE(CFGOPT_REPO1_S3_TOKEN_STR); +#define CFGOPT_REPO1_S3_URI_STYLE "repo1-s3-uri-style" + STRING_DECLARE(CFGOPT_REPO1_S3_URI_STYLE_STR); #define CFGOPT_REPO1_S3_VERIFY_TLS "repo1-s3-verify-tls" STRING_DECLARE(CFGOPT_REPO1_S3_VERIFY_TLS_STR); #define CFGOPT_REPO1_TYPE "repo1-type" @@ -398,7 +400,7 @@ Option constants #define CFGOPT_TYPE "type" STRING_DECLARE(CFGOPT_TYPE_STR); -#define CFG_OPTION_TOTAL 172 +#define CFG_OPTION_TOTAL 173 /*********************************************************************************************************************************** Command enum @@ -588,6 +590,7 @@ typedef enum cfgOptRepoS3Port, cfgOptRepoS3Region, cfgOptRepoS3Token, + cfgOptRepoS3UriStyle, cfgOptRepoS3VerifyTls, cfgOptRepoType, cfgOptResume, diff --git a/src/config/define.auto.c b/src/config/define.auto.c index a8f4ad1cf..b28473550 100644 --- a/src/config/define.auto.c +++ b/src/config/define.auto.c @@ -3938,6 +3938,68 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST ) ) + // ----------------------------------------------------------------------------------------------------------------------------- + CFGDEFDATA_OPTION + ( + CFGDEFDATA_OPTION_NAME("repo-s3-uri-style") + CFGDEFDATA_OPTION_REQUIRED(true) + CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal) + CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString) + CFGDEFDATA_OPTION_INTERNAL(false) + + CFGDEFDATA_OPTION_INDEX_TOTAL(1) + CFGDEFDATA_OPTION_SECURE(false) + + CFGDEFDATA_OPTION_HELP_SECTION("repository") + CFGDEFDATA_OPTION_HELP_SUMMARY("S3 URI Style.") + CFGDEFDATA_OPTION_HELP_DESCRIPTION + ( + "The following URI styles are supported:\n" + "\n" + "* host - Connect to bucket.endpoint host.\n" + "* path - Connect to endpoint host and prepend bucket to URIs." + ) + + CFGDEFDATA_OPTION_COMMAND_LIST + ( + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePushAsync) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdLocal) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdLs) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRemote) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStart) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStop) + ) + + CFGDEFDATA_OPTION_OPTIONAL_LIST + ( + CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST + ( + "host", + "path" + ) + + CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST + ( + cfgDefOptRepoType, + "s3" + ) + + CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("host") + CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo") + ) + ) + // ----------------------------------------------------------------------------------------------------------------------------- CFGDEFDATA_OPTION ( diff --git a/src/config/define.auto.h b/src/config/define.auto.h index e8cabf793..8222332db 100644 --- a/src/config/define.auto.h +++ b/src/config/define.auto.h @@ -131,6 +131,7 @@ typedef enum cfgDefOptRepoS3Port, cfgDefOptRepoS3Region, cfgDefOptRepoS3Token, + cfgDefOptRepoS3UriStyle, cfgDefOptRepoS3VerifyTls, cfgDefOptRepoType, cfgDefOptResume, diff --git a/src/config/parse.auto.c b/src/config/parse.auto.c index be80260e4..527bff7aa 100644 --- a/src/config/parse.auto.c +++ b/src/config/parse.auto.c @@ -2120,6 +2120,18 @@ static const struct option optionList[] = .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3Token, }, + // repo-s3-uri-style option + // ----------------------------------------------------------------------------------------------------------------------------- + { + .name = CFGOPT_REPO1_S3_URI_STYLE, + .has_arg = required_argument, + .val = PARSE_OPTION_FLAG | cfgOptRepoS3UriStyle, + }, + { + .name = "reset-" CFGOPT_REPO1_S3_URI_STYLE, + .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3UriStyle, + }, + // repo-s3-verify-tls option and deprecations // ----------------------------------------------------------------------------------------------------------------------------- { @@ -2489,6 +2501,7 @@ static const ConfigOption optionResolveOrder[] = cfgOptRepoS3Port, cfgOptRepoS3Region, cfgOptRepoS3Token, + cfgOptRepoS3UriStyle, cfgOptRepoS3VerifyTls, cfgOptTarget, cfgOptTargetAction, diff --git a/src/storage/helper.c b/src/storage/helper.c index a39406ae7..0ef2bc6b5 100644 --- a/src/storage/helper.c +++ b/src/storage/helper.c @@ -369,6 +369,7 @@ storageRepoGet(const String *type, bool write) result = storageS3New( cfgOptionStr(cfgOptRepoPath), write, storageRepoPathExpression, cfgOptionStr(cfgOptRepoS3Bucket), endPoint, + strEqZ(cfgOptionStr(cfgOptRepoS3UriStyle), STORAGE_S3_URI_STYLE_HOST) ? storageS3UriStyleHost : storageS3UriStylePath, cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key), cfgOptionStr(cfgOptRepoS3KeySecret), cfgOptionTest(cfgOptRepoS3Token) ? cfgOptionStr(cfgOptRepoS3Token) : NULL, STORAGE_S3_PARTSIZE_MIN, STORAGE_S3_DELETE_MAX, host, port, STORAGE_S3_TIMEOUT_DEFAULT, cfgOptionBool(cfgOptRepoS3VerifyTls), diff --git a/src/storage/s3/storage.c b/src/storage/s3/storage.c index 2b5e165a6..c6f1d84de 100644 --- a/src/storage/s3/storage.c +++ b/src/storage/s3/storage.c @@ -98,6 +98,7 @@ struct StorageS3 const String *securityToken; // Security token, if any size_t partSize; // Part size for multi-part upload unsigned int deleteMax; // Maximum objects that can be deleted in one request + StorageS3UriStyle uriStyle; // Path or host style URIs const String *bucketEndpoint; // Set to {bucket}.{endpoint} unsigned int port; // Host port @@ -258,6 +259,10 @@ storageS3Request( unsigned int retryRemaining = 2; bool done; + // When using path-style URIs the bucket name needs to be prepended + if (this->uriStyle == storageS3UriStylePath) + uri = strNewFmt("/%s%s", strPtr(this->bucket), strPtr(uri)); + do { done = true; @@ -917,9 +922,9 @@ static const StorageInterface storageInterfaceS3 = Storage * storageS3New( const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, - const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey, - const String *securityToken, size_t partSize, unsigned int deleteMax, const String *host, unsigned int port, TimeMSec timeout, - bool verifyPeer, const String *caFile, const String *caPath) + const String *endPoint, StorageS3UriStyle uriStyle, const String *region, const String *accessKey, + const String *secretAccessKey, const String *securityToken, size_t partSize, unsigned int deleteMax, const String *host, + unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, path); @@ -927,6 +932,7 @@ storageS3New( FUNCTION_LOG_PARAM(FUNCTIONP, pathExpressionFunction); FUNCTION_LOG_PARAM(STRING, bucket); FUNCTION_LOG_PARAM(STRING, endPoint); + FUNCTION_LOG_PARAM(ENUM, uriStyle); FUNCTION_LOG_PARAM(STRING, region); FUNCTION_TEST_PARAM(STRING, accessKey); FUNCTION_TEST_PARAM(STRING, secretAccessKey); @@ -962,7 +968,9 @@ storageS3New( driver->securityToken = strDup(securityToken); driver->partSize = partSize; driver->deleteMax = deleteMax; - driver->bucketEndpoint = strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint)); + driver->uriStyle = uriStyle; + driver->bucketEndpoint = uriStyle == storageS3UriStyleHost ? + strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint)) : strDup(endPoint); driver->port = port; // Force the signing key to be generated on the first run diff --git a/src/storage/s3/storage.h b/src/storage/s3/storage.h index f7f2b570c..d7c63f566 100644 --- a/src/storage/s3/storage.h +++ b/src/storage/s3/storage.h @@ -12,6 +12,18 @@ Storage type #define STORAGE_S3_TYPE "s3" STRING_DECLARE(STORAGE_S3_TYPE_STR); +/*********************************************************************************************************************************** +URI style +***********************************************************************************************************************************/ +typedef enum +{ + storageS3UriStyleHost, + storageS3UriStylePath, +} StorageS3UriStyle; + +#define STORAGE_S3_URI_STYLE_HOST "host" +#define STORAGE_S3_URI_STYLE_PATH "path" + /*********************************************************************************************************************************** Defaults ***********************************************************************************************************************************/ @@ -24,8 +36,8 @@ Constructor ***********************************************************************************************************************************/ Storage *storageS3New( const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, - const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey, - const String *securityToken, size_t partSize, unsigned int deleteMax, const String *host, unsigned int port, TimeMSec timeout, - bool verifyPeer, const String *caFile, const String *caPath); + const String *endPoint, StorageS3UriStyle uriStyle, const String *region, const String *accessKey, + const String *secretAccessKey, const String *securityToken, size_t partSize, unsigned int deleteMax, const String *host, + unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath); #endif diff --git a/test/src/module/command/helpTest.c b/test/src/module/command/helpTest.c index a0b661f6e..9006147e0 100644 --- a/test/src/module/command/helpTest.c +++ b/test/src/module/command/helpTest.c @@ -210,6 +210,7 @@ testRun(void) " --repo-s3-port s3 repository port [default=443]\n" " --repo-s3-region s3 repository region\n" " --repo-s3-token s3 repository security token\n" + " --repo-s3-uri-style s3 URI Style [default=host]\n" " --repo-s3-verify-tls verify S3 server certificate [default=y]\n" " --repo-type type of storage used for the repository\n" " [default=posix]\n" diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index dff7a94bb..ca1aa3c8e 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -9,14 +9,14 @@ Test S3 Storage /*********************************************************************************************************************************** Test server ***********************************************************************************************************************************/ -#define S3_TEST_HOST "bucket.s3.amazonaws.com" +#define S3_TEST_HOST "s3.amazonaws.com" #define DATE_REPLACE "????????" #define DATETIME_REPLACE "????????T??????Z" #define SHA256_REPLACE \ "????????????????????????????????????????????????????????????????" static const char * -testS3ServerRequest(const char *verb, const char *uri, const char *content) +testS3ServerRequest(const char *verb, const char *uri, const char *content, StorageS3UriStyle uriStyle) { String *request = strNewFmt( "%s %s HTTP/1.1\r\n" @@ -40,9 +40,13 @@ testS3ServerRequest(const char *verb, const char *uri, const char *content) strCatFmt(request, "content-md5:%s\r\n", md5Hash); } + if (uriStyle == storageS3UriStyleHost) + strCatFmt(request, "host:bucket." S3_TEST_HOST "\r\n"); + else + strCatFmt(request, "host:" S3_TEST_HOST "\r\n"); + strCatFmt( request, - "host:" S3_TEST_HOST "\r\n" "x-amz-content-sha256:%s\r\n" "x-amz-date:" DATETIME_REPLACE "\r\n" "\r\n", @@ -88,29 +92,29 @@ testS3Server(void) // storageS3NewRead() and StorageS3FileRead // ------------------------------------------------------------------------------------------------------------------------- // Ignore missing file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/fi%26le.txt", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/fi%26le.txt", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); // Error on missing file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); // Get file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, "this is a sample file")); // Get zero-length file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file0.txt", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file0.txt", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); // Throw non-404 error - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(303, "Some bad status", NULL, "CONTENT")); // storageS3NewWrite() and StorageWriteS3 // ------------------------------------------------------------------------------------------------------------------------- // File is written all at once - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD")); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD", storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 403, "Forbidden", NULL, "" @@ -125,15 +129,15 @@ testS3Server(void) "")); harnessTlsServerAccept(); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD")); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD", storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); // Zero-length file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "")); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "", storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); // File is written in chunks with nothing left over on close - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 200, "OK", NULL, "" @@ -143,9 +147,11 @@ testS3Server(void) "WxRt" "")); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", "1234567890123456")); + harnessTlsServerExpect( + testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", "1234567890123456", storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", "etag:WxRt1", NULL)); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", "7890123456789012")); + harnessTlsServerExpect( + testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", "7890123456789012", storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", "eTag:WxRt2", NULL)); harnessTlsServerExpect(testS3ServerRequest( @@ -154,11 +160,12 @@ testS3Server(void) "" "1WxRt1" "2WxRt2" - "\n")); + "\n", + storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); // File is written in chunks with something left over on close - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 200, "OK", NULL, "" @@ -168,9 +175,11 @@ testS3Server(void) "RR55" "")); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", "1234567890123456")); + harnessTlsServerExpect( + testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", "1234567890123456", storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", "etag:RR551", NULL)); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", "7890")); + harnessTlsServerExpect( + testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", "7890", storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", "eTag:RR552", NULL)); harnessTlsServerExpect(testS3ServerRequest( @@ -179,27 +188,28 @@ testS3Server(void) "" "1RR551" "2RR552" - "\n")); + "\n", + storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); // storageDriverExists() // ------------------------------------------------------------------------------------------------------------------------- // File missing - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); // File exists - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:999", NULL)); // Info() // ------------------------------------------------------------------------------------------------------------------------- // File missing - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); // File exists - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 200, "OK", "content-length:9999\r\n" @@ -209,7 +219,7 @@ testS3Server(void) // InfoList() // ------------------------------------------------------------------------------------------------------------------------- harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL)); + testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStyleHost)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -228,10 +238,10 @@ testS3Server(void) // storageDriverList() // ------------------------------------------------------------------------------------------------------------------------- // Throw errors - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 344, "Another bad status", NULL, NULL)); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 344, "Another bad status with xml", NULL, "" @@ -239,7 +249,7 @@ testS3Server(void) "SomeOtherCode" "")); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 403, "Forbidden", NULL, "" @@ -247,7 +257,7 @@ testS3Server(void) "RequestTimeTooSkewed" "The difference between the request time and the current time is too large." "")); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 403, "Forbidden", NULL, "" @@ -255,7 +265,7 @@ testS3Server(void) "RequestTimeTooSkewed" "The difference between the request time and the current time is too large." "")); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); harnessTlsServerReply(testS3ServerResponse( 403, "Forbidden", NULL, "" @@ -265,7 +275,7 @@ testS3Server(void) "")); // list a file/path in root - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -280,7 +290,8 @@ testS3Server(void) "")); // list a file in root with expression - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test", NULL)); + harnessTlsServerExpect( + testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test", NULL, storageS3UriStyleHost)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -293,7 +304,7 @@ testS3Server(void) // list files with continuation harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL)); + testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStyleHost)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -316,7 +327,7 @@ testS3Server(void) HTTP_VERB_GET, "/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2" "&prefix=path%2Fto%2F", - NULL)); + NULL, storageS3UriStyleHost)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -332,7 +343,7 @@ testS3Server(void) // list files with expression harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest", NULL)); + testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest", NULL, storageS3UriStyleHost)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -357,8 +368,13 @@ testS3Server(void) // storageDriverPathRemove() // ------------------------------------------------------------------------------------------------------------------------- + // Switch to path-style URIs + harnessTlsServerClose(); + harnessTlsServerAccept(); + // delete files from root - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2", NULL)); + harnessTlsServerExpect( + testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2", NULL, storageS3UriStylePath)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -373,18 +389,20 @@ testS3Server(void) "")); harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_POST, "/?delete=", + testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=", "\n" "true" "test1.txt" "path1/xxx.zzz" - "\n")); + "\n", + storageS3UriStylePath)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, "")); // nothing to do in empty subpath - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2F", NULL)); + harnessTlsServerExpect( + testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F", NULL, storageS3UriStylePath)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -394,7 +412,7 @@ testS3Server(void) // delete with continuation harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2Fto%2F", NULL)); + testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStylePath)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -410,7 +428,9 @@ testS3Server(void) "")); harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F", NULL)); + testS3ServerRequest( + HTTP_VERB_GET, "/bucket/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F", NULL, + storageS3UriStylePath)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -425,25 +445,27 @@ testS3Server(void) "")); harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_POST, "/?delete=", + testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=", "\n" "true" "path/to/test1.txt" "path/to/test3.txt" - "\n")); + "\n", + storageS3UriStylePath)); harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_POST, "/?delete=", + testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=", "\n" "true" "path/to/test2.txt" - "\n")); + "\n", + storageS3UriStylePath)); harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); // delete error harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=path%2F", NULL)); + testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F", NULL, storageS3UriStylePath)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -458,12 +480,13 @@ testS3Server(void) "")); harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_POST, "/?delete=", + testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=", "\n" "true" "path/sample.txt" "path/sample2.txt" - "\n")); + "\n", + storageS3UriStylePath)); harnessTlsServerReply( testS3ServerResponse( 200, "OK", NULL, @@ -475,7 +498,7 @@ testS3Server(void) // storageDriverRemove() // ------------------------------------------------------------------------------------------------------------------------- // remove file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_DELETE, "/path/to/test.txt", NULL)); + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_DELETE, "/bucket/path/to/test.txt", NULL, storageS3UriStylePath)); harnessTlsServerReply(testS3ServerResponse(204, "No Content", NULL, NULL)); harnessTlsServerClose(); @@ -669,8 +692,8 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- StorageS3 *driver = (StorageS3 *)storageDriver( storageS3New( - path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, 16, 2, NULL, 0, 0, testContainer(), - NULL, NULL)); + path, true, NULL, bucket, endPoint, storageS3UriStyleHost, region, accessKey, secretAccessKey, NULL, 16, 2, NULL, 0, + 0, testContainer(), NULL, NULL)); HttpHeader *header = httpHeaderNew(NULL); @@ -717,8 +740,8 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- driver = (StorageS3 *)storageDriver( storageS3New( - path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, securityToken, 16, 2, NULL, 0, 0, - testContainer(), NULL, NULL)); + path, true, NULL, bucket, endPoint, storageS3UriStyleHost, region, accessKey, secretAccessKey, securityToken, 16, 2, + NULL, 0, 0, testContainer(), NULL, NULL)); TEST_RESULT_VOID( storageS3Auth(driver, strNew("GET"), strNew("/"), query, strNew("20170606T121212Z"), header, HASH_TYPE_SHA256_ZERO_STR), @@ -737,8 +760,8 @@ testRun(void) testS3Server(); Storage *s3 = storageS3New( - path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, 16, 2, host, port, 1000, testContainer(), - NULL, NULL); + path, true, NULL, bucket, endPoint, storageS3UriStyleHost, region, accessKey, secretAccessKey, NULL, 16, 2, host, port, + 1000, testContainer(), NULL, NULL); // Coverage for noop functions // ------------------------------------------------------------------------------------------------------------------------- @@ -767,7 +790,7 @@ testRun(void) "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" - "host: " S3_TEST_HOST "\n" + "host: bucket." S3_TEST_HOST "\n" "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" "x-amz-date: \n" "*** Response Headers ***:\n" @@ -854,7 +877,7 @@ testRun(void) "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" - "host: " S3_TEST_HOST "\n" + "host: bucket." S3_TEST_HOST "\n" "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" "x-amz-date: "); TEST_ERROR(storageListP(s3, strNew("/")), ProtocolError, @@ -864,7 +887,7 @@ testRun(void) "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" - "host: " S3_TEST_HOST "\n" + "host: bucket." S3_TEST_HOST "\n" "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" "x-amz-date: \n" "*** Response Headers ***:\n" @@ -878,7 +901,7 @@ testRun(void) "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" - "host: " S3_TEST_HOST "\n" + "host: bucket." S3_TEST_HOST "\n" "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" "x-amz-date: \n" "*** Response Headers ***:\n" @@ -900,6 +923,11 @@ testRun(void) // storageDriverPathRemove() // ------------------------------------------------------------------------------------------------------------------------- + // Switch to path-style URIs + s3 = storageS3New( + path, true, NULL, bucket, endPoint, storageS3UriStylePath, region, accessKey, secretAccessKey, NULL, 16, 2, host, port, + 1000, testContainer(), NULL, NULL); + TEST_ERROR( storagePathRemoveP(s3, strNew("/")), AssertError, "assertion 'param.recurse || storageFeature(this, storageFeaturePath)' failed");