From 256b727a3d13b680ab2a4c7b3669649c857d1daf Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 21 Nov 2018 19:32:49 -0500 Subject: [PATCH] Add S3 storage driver. Only the storageNewRead() and storageList() functions are currently implemented, but this is enough to enable S3 for the archive-get command. --- doc/xml/release.xml | 4 + src/Makefile | 10 +- src/storage/driver/s3/fileRead.c | 190 +++++++++ src/storage/driver/s3/fileRead.h | 50 +++ src/storage/driver/s3/storage.c | 702 +++++++++++++++++++++++++++++++ src/storage/driver/s3/storage.h | 75 ++++ src/storage/helper.c | 13 + src/storage/storage.c | 4 +- test/define.yaml | 10 + test/src/module/storage/s3Test.c | 419 ++++++++++++++++++ 10 files changed, 1474 insertions(+), 3 deletions(-) create mode 100644 src/storage/driver/s3/fileRead.c create mode 100644 src/storage/driver/s3/fileRead.h create mode 100644 src/storage/driver/s3/storage.c create mode 100644 src/storage/driver/s3/storage.h create mode 100644 test/src/module/storage/s3Test.c diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 83ecfc16e..04d69a5a6 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -15,6 +15,10 @@ + +

Add S3 storage driver.

+ +

Add HttpClient object.

 diff --git a/src/Makefile b/src/Makefile index eac35a3f5..8c79d3324 100644 --- a/src/Makefile +++ b/src/Makefile @@ -135,6 +135,8 @@ SRCS = \ storage/driver/posix/common.c \ storage/driver/posix/fileRead.c \ storage/driver/posix/fileWrite.c \ + storage/driver/s3/fileRead.c \ + storage/driver/s3/storage.c \ storage/fileRead.c \ storage/fileWrite.c \ storage/helper.c \ @@ -397,13 +399,19 @@ storage/driver/posix/fileWrite.o: storage/driver/posix/fileWrite.c common/assert storage/driver/posix/storage.o: storage/driver/posix/storage.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h storage/driver/posix/common.h storage/driver/posix/fileRead.h storage/driver/posix/fileWrite.h storage/driver/posix/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h $(CC) $(CFLAGS) -c storage/driver/posix/storage.c -o storage/driver/posix/storage.o +storage/driver/s3/fileRead.o: storage/driver/s3/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h storage/driver/s3/fileRead.h storage/driver/s3/storage.h storage/fileRead.h storage/fileRead.intern.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h + $(CC) $(CFLAGS) -c storage/driver/s3/fileRead.c -o storage/driver/s3/fileRead.o + +storage/driver/s3/storage.o: storage/driver/s3/storage.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/common.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/type/xml.h crypto/hash.h storage/driver/s3/fileRead.h storage/driver/s3/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h + $(CC) $(CFLAGS) -c storage/driver/s3/storage.c -o storage/driver/s3/storage.o + storage/fileRead.o: storage/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/fileRead.h storage/fileRead.intern.h $(CC) $(CFLAGS) -c storage/fileRead.c -o storage/fileRead.o storage/fileWrite.o: storage/fileWrite.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/fileWrite.h storage/fileWrite.intern.h version.h $(CC) $(CFLAGS) -c storage/fileWrite.c -o storage/fileWrite.o -storage/helper.o: storage/helper.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/driver/posix/fileRead.h storage/driver/posix/fileWrite.h storage/driver/posix/storage.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h storage/storage.intern.h +storage/helper.o: storage/helper.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/driver/posix/fileRead.h storage/driver/posix/fileWrite.h storage/driver/posix/storage.h storage/driver/s3/storage.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h storage/storage.intern.h $(CC) $(CFLAGS) -c storage/helper.c -o storage/helper.o storage/storage.o: storage/storage.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h diff --git a/src/storage/driver/s3/fileRead.c b/src/storage/driver/s3/fileRead.c new file mode 100644 index 000000000..b346ee862 --- /dev/null +++ b/src/storage/driver/s3/fileRead.c @@ -0,0 +1,190 @@ +/*********************************************************************************************************************************** +S3 Storage File Read Driver +***********************************************************************************************************************************/ +#include +#include + +#include "common/assert.h" +#include "common/debug.h" +#include "common/io/http/client.h" +#include "common/io/read.intern.h" +#include "common/log.h" +#include "common/memContext.h" +#include "storage/driver/s3/fileRead.h" +#include "storage/fileRead.intern.h" + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct StorageDriverS3FileRead +{ + MemContext *memContext; + StorageDriverS3 *storage; + StorageFileRead *interface; + IoRead *io; + String *name; + bool ignoreMissing; + + HttpClient *httpClient; // Http client for requests +}; + +/*********************************************************************************************************************************** +Create a new file +***********************************************************************************************************************************/ +StorageDriverS3FileRead *storageDriverS3FileReadNew(StorageDriverS3 *storage, const String *name, bool ignoreMissing) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STRING, name); + FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing); + + FUNCTION_TEST_ASSERT(name != NULL); + FUNCTION_DEBUG_END(); + + StorageDriverS3FileRead *this = NULL; + + // Create the file object + MEM_CONTEXT_NEW_BEGIN("StorageDriverS3FileRead") + { + this = memNew(sizeof(StorageDriverS3FileRead)); + this->memContext = MEM_CONTEXT_NEW(); + this->storage = storage; + this->name = strDup(name); + this->ignoreMissing = ignoreMissing; + + this->interface = storageFileReadNewP( + strNew(STORAGE_DRIVER_S3_TYPE), this, + .ignoreMissing = (StorageFileReadInterfaceIgnoreMissing)storageDriverS3FileReadIgnoreMissing, + .io = (StorageFileReadInterfaceIo)storageDriverS3FileReadIo, + .name = (StorageFileReadInterfaceName)storageDriverS3FileReadName); + + this->io = ioReadNewP( + this, .eof = (IoReadInterfaceEof)storageDriverS3FileReadEof, + .open = (IoReadInterfaceOpen)storageDriverS3FileReadOpen, .read = (IoReadInterfaceRead)storageDriverS3FileRead); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_DEBUG_RESULT(STORAGE_DRIVER_S3_FILE_READ, this); +} + +/*********************************************************************************************************************************** +Open the file +***********************************************************************************************************************************/ +bool +storageDriverS3FileReadOpen(StorageDriverS3FileRead *this) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3_FILE_READ, this); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_ASSERT(this->httpClient == NULL); + FUNCTION_DEBUG_END(); + + bool result = false; + + // Request the file + storageDriverS3Request(this->storage, HTTP_VERB_GET_STR, this->name, NULL, NULL, false, true); + + // On success + this->httpClient = storageDriverS3HttpClient(this->storage); + + if (httpClientResponseCode(this->httpClient) == HTTP_RESPONSE_CODE_OK) + result = true; + + // Else error unless ignore missing + else if (!this->ignoreMissing) + THROW_FMT(FileMissingError, "unable to open '%s': No such file or directory", strPtr(this->name)); + + FUNCTION_DEBUG_RESULT(BOOL, result); +} + +/*********************************************************************************************************************************** +Read from a file +***********************************************************************************************************************************/ +size_t +storageDriverS3FileRead(StorageDriverS3FileRead *this, Buffer *buffer) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3_FILE_READ, this); + FUNCTION_DEBUG_PARAM(BUFFER, buffer); + + FUNCTION_DEBUG_ASSERT(this != NULL && this->httpClient != NULL); + FUNCTION_DEBUG_ASSERT(buffer != NULL && !bufFull(buffer)); + FUNCTION_DEBUG_END(); + + FUNCTION_DEBUG_RESULT(SIZE, ioRead(httpClientIoRead(this->httpClient), buffer)); +} + +/*********************************************************************************************************************************** +Has file reached EOF? +***********************************************************************************************************************************/ +bool +storageDriverS3FileReadEof(const StorageDriverS3FileRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this); + + FUNCTION_DEBUG_ASSERT(this != NULL && this->httpClient != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(BOOL, ioReadEof(httpClientIoRead(this->httpClient))); +} + +/*********************************************************************************************************************************** +Should a missing file be ignored? +***********************************************************************************************************************************/ +bool +storageDriverS3FileReadIgnoreMissing(const StorageDriverS3FileRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(BOOL, this->ignoreMissing); +} + +/*********************************************************************************************************************************** +Get the interface +***********************************************************************************************************************************/ +StorageFileRead * +storageDriverS3FileReadInterface(const StorageDriverS3FileRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(STORAGE_FILE_READ, this->interface); +} + +/*********************************************************************************************************************************** +Get the I/O interface +***********************************************************************************************************************************/ +IoRead * +storageDriverS3FileReadIo(const StorageDriverS3FileRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(IO_READ, this->io); +} + +/*********************************************************************************************************************************** +File name +***********************************************************************************************************************************/ +const String * +storageDriverS3FileReadName(const StorageDriverS3FileRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(CONST_STRING, this->name); +} diff --git a/src/storage/driver/s3/fileRead.h b/src/storage/driver/s3/fileRead.h new file mode 100644 index 000000000..81c87809c --- /dev/null +++ b/src/storage/driver/s3/fileRead.h @@ -0,0 +1,50 @@ +/*********************************************************************************************************************************** +S3 Storage File Read Driver +***********************************************************************************************************************************/ +#ifndef STORAGE_DRIVER_S3_FILEREAD_H +#define STORAGE_DRIVER_S3_FILEREAD_H + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +typedef struct StorageDriverS3FileRead StorageDriverS3FileRead; + +#include "common/type/buffer.h" +#include "common/type/string.h" +#include "storage/driver/s3/storage.h" +#include "storage/fileRead.h" + +/*********************************************************************************************************************************** +Constructor +***********************************************************************************************************************************/ +StorageDriverS3FileRead *storageDriverS3FileReadNew(StorageDriverS3 *storage, const String *name, bool ignoreMissing); + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +bool storageDriverS3FileReadOpen(StorageDriverS3FileRead *this); +size_t storageDriverS3FileRead(StorageDriverS3FileRead *this, Buffer *buffer); + +/*********************************************************************************************************************************** +Getters +***********************************************************************************************************************************/ +bool storageDriverS3FileReadEof(const StorageDriverS3FileRead *this); +bool storageDriverS3FileReadIgnoreMissing(const StorageDriverS3FileRead *this); +StorageFileRead *storageDriverS3FileReadInterface(const StorageDriverS3FileRead *this); +IoRead *storageDriverS3FileReadIo(const StorageDriverS3FileRead *this); +const String *storageDriverS3FileReadName(const StorageDriverS3FileRead *this); + +/*********************************************************************************************************************************** +Destructor +***********************************************************************************************************************************/ +void storageDriverS3FileReadFree(StorageDriverS3FileRead *this); + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +#define FUNCTION_DEBUG_STORAGE_DRIVER_S3_FILE_READ_TYPE \ + StorageDriverS3FileRead * +#define FUNCTION_DEBUG_STORAGE_DRIVER_S3_FILE_READ_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "StorageDriverS3FileRead", buffer, bufferSize) + +#endif diff --git a/src/storage/driver/s3/storage.c b/src/storage/driver/s3/storage.c new file mode 100644 index 000000000..90062921a --- /dev/null +++ b/src/storage/driver/s3/storage.c @@ -0,0 +1,702 @@ +/*********************************************************************************************************************************** +S3 Storage Driver +***********************************************************************************************************************************/ +#include + +#include "common/assert.h" +#include "common/debug.h" +#include "common/io/http/common.h" +#include "common/log.h" +#include "common/memContext.h" +#include "common/regExp.h" +#include "common/type/xml.h" +#include "crypto/hash.h" +#include "storage/driver/s3/fileRead.h" +#include "storage/driver/s3/storage.h" + +/*********************************************************************************************************************************** +Driver type constant string +***********************************************************************************************************************************/ +STRING_EXTERN(STORAGE_DRIVER_S3_TYPE_STR, STORAGE_DRIVER_S3_TYPE); + +/*********************************************************************************************************************************** +S3 http headers +***********************************************************************************************************************************/ +STRING_STATIC(S3_HEADER_AUTHORIZATION_STR, "authorization"); +STRING_STATIC(S3_HEADER_CONTENT_SHA256_STR, "x-amz-content-sha256"); +STRING_STATIC(S3_HEADER_DATE_STR, "x-amz-date"); +STRING_STATIC(S3_HEADER_HOST_STR, "host"); +STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-security-token"); + +/*********************************************************************************************************************************** +S3 query tokens +***********************************************************************************************************************************/ +STRING_STATIC(S3_QUERY_CONTINUATION_TOKEN_STR, "continuation-token"); +STRING_STATIC(S3_QUERY_DELIMITER_STR, "delimiter"); +STRING_STATIC(S3_QUERY_LIST_TYPE_STR, "list-type"); +STRING_STATIC(S3_QUERY_PREFIX_STR, "prefix"); + +STRING_STATIC(S3_QUERY_VALUE_LIST_TYPE_2_STR, "2"); + +/*********************************************************************************************************************************** +XML tags +***********************************************************************************************************************************/ +STRING_STATIC(S3_XML_TAG_COMMON_PREFIXES_STR, "CommonPrefixes"); +STRING_STATIC(S3_XML_TAG_CONTENTS_STR, "Contents"); +STRING_STATIC(S3_XML_TAG_KEY_STR, "Key"); +STRING_STATIC(S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, "NextContinuationToken"); +STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix"); + +/*********************************************************************************************************************************** +AWS authentication v4 constants +***********************************************************************************************************************************/ +#define S3 "s3" +#define AWS4 "AWS4" +#define AWS4_REQUEST "aws4_request" +#define AWS4_HMAC_SHA256 "AWS4-HMAC-SHA256" + +/*********************************************************************************************************************************** +Starting data for signing string so it will be regenerated on the first request +***********************************************************************************************************************************/ +STRING_STATIC(YYYYMMDD_STR, "YYYYMMDD"); + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct StorageDriverS3 +{ + MemContext *memContext; + Storage *interface; // Driver interface + HttpClient *httpClient; // Http client to service requests + const StringList *headerRedactList; // List of headers to redact from logging + + const String *bucket; // Bucket to store data in + const String *region; // e.g. us-east-1 + const String *accessKey; // Access key + const String *secretAccessKey; // Secret access key + const String *securityToken; // Security token, if any + const String *host; // Defaults to {bucket}.{endpoint} + + // Current signing key and date it is valid for + const String *signingKeyDate; // Date of cached signing key (so we know when to regenerate) + const Buffer *signingKey; // Cached signing key +}; + +/*********************************************************************************************************************************** +Expected ISO-8601 data/time size +***********************************************************************************************************************************/ +#define ISO_8601_DATE_TIME_SIZE 16 + +/*********************************************************************************************************************************** +Format ISO-8601 date/time for authentication +***********************************************************************************************************************************/ +static String * +storageDriverS3DateTime(time_t authTime) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(INT64, authTime); + FUNCTION_TEST_END(); + + char buffer[ISO_8601_DATE_TIME_SIZE + 1]; + + if (strftime( // {uncoverable - nothing invalid can be passed} + buffer, sizeof(buffer), "%Y%m%dT%H%M%SZ", gmtime(&authTime)) != ISO_8601_DATE_TIME_SIZE) + THROW_SYS_ERROR(AssertError, "unable to format date"); // {+uncoverable} + + FUNCTION_TEST_RESULT(STRING, strNew(buffer)); +} + +/*********************************************************************************************************************************** +Generate authorization header and add it to the supplied header list + +Based on the excellent documentation at http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html. +***********************************************************************************************************************************/ +static void +storageDriverS3Auth( + StorageDriverS3 *this, const String *verb, const String *uri, const HttpQuery *query, const String *dateTime, + HttpHeader *httpHeader, const String *payloadHash) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_TEST_PARAM(STRING, verb); + FUNCTION_TEST_PARAM(STRING, uri); + FUNCTION_TEST_PARAM(HTTP_QUERY, query); + FUNCTION_TEST_PARAM(STRING, dateTime); + FUNCTION_TEST_PARAM(KEY_VALUE, httpHeader); + FUNCTION_TEST_PARAM(STRING, payloadHash); + + FUNCTION_TEST_ASSERT(verb != NULL); + FUNCTION_TEST_ASSERT(uri != NULL); + FUNCTION_TEST_ASSERT(dateTime != NULL); + FUNCTION_TEST_ASSERT(httpHeader != NULL); + FUNCTION_TEST_ASSERT(payloadHash != NULL); + FUNCTION_TEST_END(); + + MEM_CONTEXT_TEMP_BEGIN() + { + // Get date from datetime + const String *date = strSubN(dateTime, 0, 8); + + // Set required headers + httpHeaderPut(httpHeader, S3_HEADER_CONTENT_SHA256_STR, payloadHash); + httpHeaderPut(httpHeader, S3_HEADER_DATE_STR, dateTime); + httpHeaderPut(httpHeader, S3_HEADER_HOST_STR, this->host); + + if (this->securityToken != NULL) + httpHeaderPut(httpHeader, S3_HEADER_TOKEN_STR, this->securityToken); + + // Generate canonical request and signed headers + const StringList *headerList = strLstSort(strLstDup(httpHeaderList(httpHeader)), sortOrderAsc); + String *signedHeaders = NULL; + + String *canonicalRequest = strNewFmt( + "%s\n%s\n%s\n", strPtr(verb), strPtr(uri), query == NULL ? "" : strPtr(httpQueryRender(query))); + + for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++) + { + const String *headerKey = strLstGet(headerList, headerIdx); + const String *headerKeyLower = strLower(strDup(headerKey)); + + // Skip the authorization header -- if it exists this is a retry + if (strEq(headerKeyLower, S3_HEADER_AUTHORIZATION_STR)) + continue; + + strCatFmt(canonicalRequest, "%s:%s\n", strPtr(headerKeyLower), strPtr(httpHeaderGet(httpHeader, headerKey))); + + if (signedHeaders == NULL) + signedHeaders = strDup(headerKeyLower); + else + strCatFmt(signedHeaders, ";%s", strPtr(headerKeyLower)); + } + + strCatFmt(canonicalRequest, "\n%s\n%s", strPtr(signedHeaders), strPtr(payloadHash)); + + // Generate string to sign + const String *stringToSign = strNewFmt( + AWS4_HMAC_SHA256 "\n%s\n%s/%s/" S3 "/" AWS4_REQUEST "\n%s", strPtr(dateTime), strPtr(date), strPtr(this->region), + strPtr(bufHex(cryptoHashOneStr(HASH_TYPE_SHA256_STR, canonicalRequest)))); + + // Generate signing key. This key only needs to be regenerated every seven days but we'll do it once a day to keep the + // logic simple. It's a relatively expensive operation so we'd rather not do it for every request. + // If the cached signing key has expired (or has noe been generated) then regenerate it + if (!strEq(date, this->signingKeyDate)) + { + const Buffer *dateKey = cryptoHmacOne( + HASH_TYPE_SHA256_STR, bufNewStr(strNewFmt(AWS4 "%s", strPtr(this->secretAccessKey))), bufNewStr(date)); + const Buffer *regionKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, dateKey, bufNewStr(this->region)); + const Buffer *serviceKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, regionKey, bufNewZ(S3)); + + // Switch to the object context so signing key and date are not lost + MEM_CONTEXT_BEGIN(this->memContext) + { + this->signingKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, serviceKey, bufNewZ(AWS4_REQUEST)); + this->signingKeyDate = strDup(date); + } + MEM_CONTEXT_END(); + } + + // Generate authorization header + const String *authorization = strNewFmt( + AWS4_HMAC_SHA256 " Credential=%s/%s/%s/" S3 "/" AWS4_REQUEST ",SignedHeaders=%s,Signature=%s", + strPtr(this->accessKey), strPtr(date), strPtr(this->region), strPtr(signedHeaders), + strPtr(bufHex(cryptoHmacOne(HASH_TYPE_SHA256_STR, this->signingKey, bufNewStr(stringToSign))))); + + httpHeaderPut(httpHeader, S3_HEADER_AUTHORIZATION_STR, authorization); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_TEST_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +New object +***********************************************************************************************************************************/ +StorageDriverS3 * +storageDriverS3New( + const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, + const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey, + const String *securityToken, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, + const String *caPath) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STRING, bucket); + FUNCTION_DEBUG_PARAM(STRING, endPoint); + FUNCTION_DEBUG_PARAM(STRING, region); + FUNCTION_TEST_PARAM(STRING, accessKey); + FUNCTION_TEST_PARAM(STRING, secretAccessKey); + FUNCTION_TEST_PARAM(STRING, securityToken); + FUNCTION_DEBUG_PARAM(STRING, host); + FUNCTION_DEBUG_PARAM(UINT, port); + FUNCTION_DEBUG_PARAM(TIME_MSEC, timeout); + FUNCTION_DEBUG_PARAM(BOOL, verifyPeer); + FUNCTION_DEBUG_PARAM(STRING, caFile); + FUNCTION_DEBUG_PARAM(STRING, caPath); + + FUNCTION_TEST_ASSERT(bucket != NULL); + FUNCTION_TEST_ASSERT(endPoint != NULL); + FUNCTION_TEST_ASSERT(region != NULL); + FUNCTION_TEST_ASSERT(accessKey != NULL); + FUNCTION_TEST_ASSERT(secretAccessKey != NULL); + FUNCTION_DEBUG_END(); + + // Create the object + StorageDriverS3 *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("StorageDriverS3") + { + this = memNew(sizeof(StorageDriverS3)); + this->memContext = MEM_CONTEXT_NEW(); + + this->bucket = strDup(bucket); + this->region = strDup(region); + this->accessKey = strDup(accessKey); + this->secretAccessKey = strDup(secretAccessKey); + this->securityToken = strDup(securityToken); + this->host = host == NULL ? strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint)) : strDup(host); + + // Force the signing key to be generated on the first run + this->signingKeyDate = YYYYMMDD_STR; + + // Create the storage interface + this->interface = storageNewP( + STORAGE_DRIVER_S3_TYPE_STR, path, 0, 0, write, pathExpressionFunction, this, + .exists = (StorageInterfaceExists)storageDriverS3Exists, .info = (StorageInterfaceInfo)storageDriverS3Info, + .list = (StorageInterfaceList)storageDriverS3List, .newRead = (StorageInterfaceNewRead)storageDriverS3NewRead, + .newWrite = (StorageInterfaceNewWrite)storageDriverS3NewWrite, + .pathCreate = (StorageInterfacePathCreate)storageDriverS3PathCreate, + .pathRemove = (StorageInterfacePathRemove)storageDriverS3PathRemove, + .pathSync = (StorageInterfacePathSync)storageDriverS3PathSync, .remove = (StorageInterfaceRemove)storageDriverS3Remove); + + // Create the http client used to service requests + this->httpClient = httpClientNew(this->host, port, timeout, verifyPeer, caFile, caPath); + this->headerRedactList = strLstAdd(strLstNew(), S3_HEADER_AUTHORIZATION_STR); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_DEBUG_RESULT(STORAGE_DRIVER_S3, this); +} + +/*********************************************************************************************************************************** +Process S3 request +***********************************************************************************************************************************/ +Buffer * +storageDriverS3Request( + StorageDriverS3 *this, const String *verb, const String *uri, const HttpQuery *query, const Buffer *body, bool returnContent, + bool allowMissing) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, verb); + FUNCTION_DEBUG_PARAM(STRING, uri); + FUNCTION_DEBUG_PARAM(HTTP_QUERY, query); + FUNCTION_DEBUG_PARAM(BUFFER, body); + FUNCTION_DEBUG_PARAM(BOOL, returnContent); + FUNCTION_DEBUG_PARAM(BOOL, allowMissing); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_ASSERT(verb != NULL); + FUNCTION_TEST_ASSERT(uri != NULL); + FUNCTION_DEBUG_END(); + + Buffer *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + // Create header list and add content length + HttpHeader *requestHeader = httpHeaderNew(this->headerRedactList); + httpHeaderAdd(requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR); + + // Generate authorization header + storageDriverS3Auth( + this, verb, uri, query, storageDriverS3DateTime(time(NULL)), requestHeader, + strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")); + + // Process request + result = httpClientRequest(this->httpClient, verb, uri, query, requestHeader, returnContent); + + // Error if the request was not successful + if (httpClientResponseCode(this->httpClient) != HTTP_RESPONSE_CODE_OK && + (!allowMissing || httpClientResponseCode(this->httpClient) != HTTP_RESPONSE_CODE_NOT_FOUND)) + { + // General error message + String *error = strNewFmt( + "S3 request failed with %u: %s", httpClientResponseCode(this->httpClient), + strPtr(httpClientResponseMessage(this->httpClient))); + + // Output uri/query + strCat(error, "\n*** URI/Query ***:"); + + strCatFmt(error, "\n%s", strPtr(httpUriEncode(uri, true))); + + if (query != NULL) + strCatFmt(error, "?%s", strPtr(httpQueryRender(query))); + + // Output request headers + const StringList *requestHeaderList = httpHeaderList(requestHeader); + + strCat(error, "\n*** Request Headers ***:"); + + for (unsigned int requestHeaderIdx = 0; requestHeaderIdx < strLstSize(requestHeaderList); requestHeaderIdx++) + { + const String *key = strLstGet(requestHeaderList, requestHeaderIdx); + + strCatFmt( + error, "\n%s: %s", strPtr(key), + httpHeaderRedact(requestHeader, key) || strEq(key, S3_HEADER_DATE_STR) ? + "" : strPtr(httpHeaderGet(requestHeader, key))); + } + + // Output response headers + const HttpHeader *responseHeader = httpClientReponseHeader(this->httpClient); + const StringList *responseHeaderList = httpHeaderList(responseHeader); + + if (strLstSize(responseHeaderList) > 0) + { + strCat(error, "\n*** Response Headers ***:"); + + for (unsigned int responseHeaderIdx = 0; responseHeaderIdx < strLstSize(responseHeaderList); responseHeaderIdx++) + { + const String *key = strLstGet(responseHeaderList, responseHeaderIdx); + strCatFmt(error, "\n%s: %s", strPtr(key), strPtr(httpHeaderGet(responseHeader, key))); + } + } + + // If there was content then output it + if (result != NULL) + strCatFmt(error, "\n*** Response Content ***:\n%s", strPtr(strNewBuf(result))); + + THROW(ProtocolError, strPtr(error)); + } + + // On success move the buffer to the calling context + bufMove(result, MEM_CONTEXT_OLD()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_DEBUG_RESULT(BUFFER, result); +} + +/*********************************************************************************************************************************** +Does a file/path exist? +***********************************************************************************************************************************/ +bool +storageDriverS3Exists(StorageDriverS3 *this, const String *path) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, path); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(path != NULL); + FUNCTION_DEBUG_END(); + + bool result = false; + + THROW(AssertError, "NOT YET IMPLEMENTED"); + + FUNCTION_DEBUG_RESULT(BOOL, result); +} + +/*********************************************************************************************************************************** +File/path info +***********************************************************************************************************************************/ +StorageInfo +storageDriverS3Info(StorageDriverS3 *this, const String *file, bool ignoreMissing) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, file); + FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(file != NULL); + FUNCTION_DEBUG_END(); + + THROW(AssertError, "NOT YET IMPLEMENTED"); + + FUNCTION_DEBUG_RESULT(STORAGE_INFO, (StorageInfo){0}); +} + +/*********************************************************************************************************************************** +Get a list of files from a directory +***********************************************************************************************************************************/ +StringList * +storageDriverS3List(StorageDriverS3 *this, const String *path, bool errorOnMissing, const String *expression) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, path); + FUNCTION_DEBUG_PARAM(BOOL, errorOnMissing); + FUNCTION_DEBUG_PARAM(STRING, expression); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(path != NULL); + FUNCTION_TEST_ASSERT(!errorOnMissing); + FUNCTION_DEBUG_END(); + + StringList *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + result = strLstNew(); + const String *continuationToken = NULL; + + // Prepare regexp if an expression was passed + RegExp *regExp = (expression == NULL) ? NULL : regExpNew(expression); + + // Build the base prefix by stripping of the initial / + const String *basePrefix; + + if (strSize(path) == 1) + basePrefix = EMPTY_STR; + else + basePrefix = strNewFmt("%s/", strPtr(strSub(path, 1))); + + // Get the expression prefix when possible to limit initial results + const String *expressionPrefix = regExpPrefix(expression); + + // If there is an expression prefix then use it to build the query prefix, otherwise query prefix is base prefix + const String *queryPrefix; + + if (expressionPrefix == NULL) + queryPrefix = basePrefix; + else + { + if (strEmpty(basePrefix)) + queryPrefix = expressionPrefix; + else + queryPrefix = strNewFmt("%s%s", strPtr(basePrefix), strPtr(expressionPrefix)); + } + + // Loop as long as a continuation token returned + do + { + // Use an inner mem context here because we could potentially be retrieving millions of files so it is a good idea to + // free memory at regular intervals + MEM_CONTEXT_TEMP_BEGIN() + { + HttpQuery *query = httpQueryNew(); + + // Add continuation token from the prior loop if any + if (continuationToken != NULL) + httpQueryAdd(query, S3_QUERY_CONTINUATION_TOKEN_STR, continuationToken); + + // Add the delimiter so we don't recurse + httpQueryAdd(query, S3_QUERY_DELIMITER_STR, FSLASH_STR); + + // Use list type 2 + httpQueryAdd(query, S3_QUERY_LIST_TYPE_STR, S3_QUERY_VALUE_LIST_TYPE_2_STR); + + // Don't specified empty prefix because it is the default + if (!strEmpty(queryPrefix)) + httpQueryAdd(query, S3_QUERY_PREFIX_STR, queryPrefix); + + XmlNode *xmlRoot = xmlDocumentRoot( + xmlDocumentNewBuf(storageDriverS3Request(this, HTTP_VERB_GET_STR, FSLASH_STR, query, NULL, true, false))); + + // Get subpath list + XmlNodeList *subPathList = xmlNodeChildList(xmlRoot, S3_XML_TAG_COMMON_PREFIXES_STR); + + for (unsigned int subPathIdx = 0; subPathIdx < xmlNodeLstSize(subPathList); subPathIdx++) + { + // Get subpath name + const String *subPath = xmlNodeContent( + xmlNodeChild(xmlNodeLstGet(subPathList, subPathIdx), S3_XML_TAG_PREFIX_STR, true)); + + // Strip off base prefix and final / + subPath = strSubN(subPath, strSize(basePrefix), strSize(subPath) - strSize(basePrefix) - 1); + + // Add to list after checking expression if present + if (regExp == NULL || regExpMatch(regExp, subPath)) + strLstAdd(result, subPath); + } + + // Get file list + XmlNodeList *fileList = xmlNodeChildList(xmlRoot, S3_XML_TAG_CONTENTS_STR); + + for (unsigned int fileIdx = 0; fileIdx < xmlNodeLstSize(fileList); fileIdx++) + { + // Get file name + const String *file = xmlNodeContent(xmlNodeChild(xmlNodeLstGet(fileList, fileIdx), S3_XML_TAG_KEY_STR, true)); + + // Strip off the base prefix when present + file = strEmpty(basePrefix) ? file : strSub(file, strSize(basePrefix)); + + // Add to list after checking expression if present + if (regExp == NULL || regExpMatch(regExp, file)) + strLstAdd(result, file); + } + + // Get the continuation token and store it in the outer temp context + memContextSwitch(MEM_CONTEXT_OLD()); + continuationToken = xmlNodeContent(xmlNodeChild(xmlRoot, S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, false)); + memContextSwitch(MEM_CONTEXT_TEMP()); + } + MEM_CONTEXT_TEMP_END(); + } + while (continuationToken != NULL); + + strLstMove(result, MEM_CONTEXT_OLD()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_DEBUG_RESULT(STRING_LIST, result); +} + +/*********************************************************************************************************************************** +New file read object +***********************************************************************************************************************************/ +StorageFileRead * +storageDriverS3NewRead(StorageDriverS3 *this, const String *file, bool ignoreMissing) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, file); + FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(file != NULL); + FUNCTION_DEBUG_END(); + + FUNCTION_DEBUG_RESULT( + STORAGE_FILE_READ, storageDriverS3FileReadInterface(storageDriverS3FileReadNew(this, file, ignoreMissing))); +} + +/*********************************************************************************************************************************** +New file write object +***********************************************************************************************************************************/ +StorageFileWrite * +storageDriverS3NewWrite( + StorageDriverS3 *this, const String *file, mode_t modeFile, mode_t modePath, bool createPath, bool syncFile, + bool syncPath, bool atomic) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, file); + FUNCTION_DEBUG_PARAM(MODE, modeFile); + FUNCTION_DEBUG_PARAM(MODE, modePath); + FUNCTION_DEBUG_PARAM(BOOL, createPath); + FUNCTION_DEBUG_PARAM(BOOL, syncFile); + FUNCTION_DEBUG_PARAM(BOOL, syncPath); + FUNCTION_DEBUG_PARAM(BOOL, atomic); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(file != NULL); + FUNCTION_TEST_ASSERT(modeFile == 0); + FUNCTION_TEST_ASSERT(modePath == 0); + FUNCTION_DEBUG_END(); + + THROW(AssertError, "NOT YET IMPLEMENTED"); + + FUNCTION_DEBUG_RESULT(STORAGE_FILE_WRITE, NULL); +} + +/*********************************************************************************************************************************** +Create a path. There are no physical paths on S3 so just return success. +***********************************************************************************************************************************/ +void +storageDriverS3PathCreate(StorageDriverS3 *this, const String *path, bool errorOnExists, bool noParentCreate, mode_t mode) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, path); + FUNCTION_DEBUG_PARAM(BOOL, errorOnExists); + FUNCTION_DEBUG_PARAM(BOOL, noParentCreate); + FUNCTION_DEBUG_PARAM(MODE, mode); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(path != NULL); + FUNCTION_TEST_ASSERT(mode == 0); + FUNCTION_DEBUG_END(); + + FUNCTION_DEBUG_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +Remove a path +***********************************************************************************************************************************/ +void +storageDriverS3PathRemove(StorageDriverS3 *this, const String *path, bool errorOnMissing, bool recurse) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, path); + FUNCTION_DEBUG_PARAM(BOOL, errorOnMissing); + FUNCTION_DEBUG_PARAM(BOOL, recurse); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(path != NULL); + FUNCTION_DEBUG_END(); + + THROW(AssertError, "NOT YET IMPLEMENTED"); + + FUNCTION_DEBUG_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +Sync a path. There's no need for this on S3 so just return success. +***********************************************************************************************************************************/ +void +storageDriverS3PathSync(StorageDriverS3 *this, const String *path, bool ignoreMissing) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, path); + FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(path != NULL); + FUNCTION_DEBUG_END(); + + FUNCTION_DEBUG_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +Remove a file +***********************************************************************************************************************************/ +void +storageDriverS3Remove(StorageDriverS3 *this, const String *file, bool errorOnMissing) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + FUNCTION_DEBUG_PARAM(STRING, file); + FUNCTION_DEBUG_PARAM(BOOL, errorOnMissing); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_DEBUG_ASSERT(file != NULL); + FUNCTION_DEBUG_END(); + + THROW(AssertError, "NOT YET IMPLEMENTED"); + + FUNCTION_DEBUG_RESULT_VOID(); +} + +/*********************************************************************************************************************************** +Get http client +***********************************************************************************************************************************/ +HttpClient * +storageDriverS3HttpClient(const StorageDriverS3 *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(HTTP_CLIENT, this->httpClient); +} + +/*********************************************************************************************************************************** +Get storage interface +***********************************************************************************************************************************/ +Storage * +storageDriverS3Interface(const StorageDriverS3 *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RESULT(STORAGE, this->interface); +} diff --git a/src/storage/driver/s3/storage.h b/src/storage/driver/s3/storage.h new file mode 100644 index 000000000..0433b2796 --- /dev/null +++ b/src/storage/driver/s3/storage.h @@ -0,0 +1,75 @@ +/*********************************************************************************************************************************** +S3 Storage Driver +***********************************************************************************************************************************/ +#ifndef STORAGE_DRIVER_S3_STORAGE_H +#define STORAGE_DRIVER_S3_STORAGE_H + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +typedef struct StorageDriverS3 StorageDriverS3; + +#include "common/io/http/client.h" +#include "common/type/string.h" +#include "storage/storage.intern.h" + +/*********************************************************************************************************************************** +Driver type constant +***********************************************************************************************************************************/ +#define STORAGE_DRIVER_S3_TYPE "s3" + STRING_DECLARE(STORAGE_DRIVER_S3_TYPE_STR); + +/*********************************************************************************************************************************** +Defaults +***********************************************************************************************************************************/ +#define STORAGE_DRIVER_S3_PORT_DEFAULT 443 +#define STORAGE_DRIVER_S3_TIMEOUT_DEFAULT 60000 + +/*********************************************************************************************************************************** +Constructor +***********************************************************************************************************************************/ +StorageDriverS3 * storageDriverS3New( + const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket, + const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey, + const String *securityToken, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, + const String *caPath); + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +bool storageDriverS3Exists(StorageDriverS3 *this, const String *path); +StorageInfo storageDriverS3Info(StorageDriverS3 *this, const String *file, bool ignoreMissing); +StringList *storageDriverS3List(StorageDriverS3 *this, const String *path, bool errorOnMissing, const String *expression); +StorageFileRead *storageDriverS3NewRead(StorageDriverS3 *this, const String *file, bool ignoreMissing); +StorageFileWrite *storageDriverS3NewWrite( + StorageDriverS3 *this, const String *file, mode_t modeFile, mode_t modePath, bool createPath, bool syncFile, bool syncPath, + bool atomic); +void storageDriverS3PathCreate(StorageDriverS3 *this, const String *path, bool errorOnExists, bool noParentCreate, mode_t mode); +void storageDriverS3PathRemove(StorageDriverS3 *this, const String *path, bool errorOnMissing, bool recurse); +void storageDriverS3PathSync(StorageDriverS3 *this, const String *path, bool ignoreMissing); +void storageDriverS3Remove(StorageDriverS3 *this, const String *file, bool errorOnMissing); + +Buffer *storageDriverS3Request( + StorageDriverS3 *this, const String *verb, const String *uri, const HttpQuery *query, const Buffer *body, bool returnContent, + bool allowMissing); + +/*********************************************************************************************************************************** +Getters +***********************************************************************************************************************************/ +HttpClient *storageDriverS3HttpClient(const StorageDriverS3 *this); +Storage *storageDriverS3Interface(const StorageDriverS3 *this); + +/*********************************************************************************************************************************** +Destructor +***********************************************************************************************************************************/ +void storageDriverS3Free(StorageDriverS3 *this); + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +#define FUNCTION_DEBUG_STORAGE_DRIVER_S3_TYPE \ + StorageDriverS3 * +#define FUNCTION_DEBUG_STORAGE_DRIVER_S3_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "StorageDriverS3", buffer, bufferSize) + +#endif diff --git a/src/storage/helper.c b/src/storage/helper.c index f7888501d..1e6f26632 100644 --- a/src/storage/helper.c +++ b/src/storage/helper.c @@ -8,6 +8,7 @@ Storage Helper #include "common/regExp.h" #include "config/config.h" #include "storage/driver/posix/storage.h" +#include "storage/driver/s3/storage.h" #include "storage/helper.h" /*********************************************************************************************************************************** @@ -189,6 +190,18 @@ storageRepoGet(const String *type, bool write) cfgOptionStr(cfgOptRepoPath), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, write, storageRepoPathExpression)); } + else if (strEqZ(type, STORAGE_TYPE_S3)) + { + result = storageDriverS3Interface( + storageDriverS3New( + cfgOptionStr(cfgOptRepoPath), write, storageRepoPathExpression, cfgOptionStr(cfgOptRepoS3Bucket), + cfgOptionStr(cfgOptRepoS3Endpoint), cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key), + cfgOptionStr(cfgOptRepoS3KeySecret), cfgOptionTest(cfgOptRepoS3Token) ? cfgOptionStr(cfgOptRepoS3Token) : NULL, + cfgOptionTest(cfgOptRepoS3Host) ? cfgOptionStr(cfgOptRepoS3Host) : NULL, + STORAGE_DRIVER_S3_PORT_DEFAULT, STORAGE_DRIVER_S3_TIMEOUT_DEFAULT, cfgOptionBool(cfgOptRepoS3VerifySsl), + cfgOptionTest(cfgOptRepoS3CaFile) ? cfgOptionStr(cfgOptRepoS3CaFile) : NULL, + cfgOptionTest(cfgOptRepoS3CaPath) ? cfgOptionStr(cfgOptRepoS3CaPath) : NULL)); + } else THROW_FMT(AssertError, "invalid storage type '%s'", strPtr(type)); diff --git a/src/storage/storage.c b/src/storage/storage.c index 405a80371..2a36b3ab8 100644 --- a/src/storage/storage.c +++ b/src/storage/storage.c @@ -48,12 +48,11 @@ storageNew( FUNCTION_DEBUG_PARAM(STORAGE_INTERFACE, interface); FUNCTION_TEST_ASSERT(type != NULL); - FUNCTION_TEST_ASSERT(path != NULL); + FUNCTION_TEST_ASSERT(path != NULL && strSize(path) >= 1 && strPtr(path)[0] == '/'); FUNCTION_TEST_ASSERT(driver != NULL); FUNCTION_TEST_ASSERT(interface.exists != NULL); FUNCTION_TEST_ASSERT(interface.info != NULL); FUNCTION_TEST_ASSERT(interface.list != NULL); - FUNCTION_TEST_ASSERT(interface.move != NULL); FUNCTION_TEST_ASSERT(interface.newRead != NULL); FUNCTION_TEST_ASSERT(interface.newWrite != NULL); FUNCTION_TEST_ASSERT(interface.pathCreate != NULL); @@ -294,6 +293,7 @@ storageMove(const Storage *this, StorageFileRead *source, StorageFileWrite *dest FUNCTION_DEBUG_PARAM(STORAGE_FILE_READ, source); FUNCTION_DEBUG_PARAM(STORAGE_FILE_WRITE, destination); + FUNCTION_TEST_ASSERT(this->interface.move != NULL); FUNCTION_TEST_ASSERT(source != NULL); FUNCTION_TEST_ASSERT(destination != NULL); FUNCTION_DEBUG_ASSERT(!storageFileReadIgnoreMissing(source)); diff --git a/test/define.yaml b/test/define.yaml index 3cdf82dc9..4a8d13119 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -527,6 +527,16 @@ unit: storage/helper: full storage/storage: full + # ---------------------------------------------------------------------------------------------------------------------------- + - name: s3 + total: 3 + + coverage: + storage/driver/s3/fileRead: full + storage/driver/s3/storage: full + storage/helper: full + storage/storage: full + # ******************************************************************************************************************************** - name: protocol diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c new file mode 100644 index 000000000..12cb4008e --- /dev/null +++ b/test/src/module/storage/s3Test.c @@ -0,0 +1,419 @@ +/*********************************************************************************************************************************** +Test S3 Storage Driver +***********************************************************************************************************************************/ +#include + +#include "common/harnessConfig.h" +#include "common/harnessTls.h" + +/*********************************************************************************************************************************** +Test server +***********************************************************************************************************************************/ +#define DATE_REPLACE "????????" +#define DATETIME_REPLACE "????????T??????Z" +#define SHA256_REPLACE \ + "????????????????????????????????????????????????????????????????" + +static const char * +testS3ServerRequest(const char *verb, const char *uri) +{ + String *request = strNewFmt( + "%s %s HTTP/1.1\r\n" + "authorization:AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/" DATE_REPLACE "/us-east-1/s3/aws4_request," + "SignedHeaders=content-length;host;x-amz-content-sha256;x-amz-date,Signature=" SHA256_REPLACE "\r\n" + "content-length:%u\r\n" + "host:" TLS_TEST_HOST "\r\n" + "x-amz-content-sha256:%s\r\n" + "x-amz-date:" DATETIME_REPLACE "\r\n" + "\r\n", + verb, uri, (unsigned int)0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + + return strPtr(request); +} + +static const char * +testS3ServerResponse(unsigned int code, const char *message, const char *content) +{ + String *response = strNewFmt("HTTP/1.1 %u %s\r\n", code, message); + + if (content != NULL) + { + strCatFmt( + response, + "content-length:%zu\r\n" + "\r\n" + "%s", + strlen(content), content); + } + else + strCat(response, "\r\n"); + + return strPtr(response); +} + +static void +testS3Server(void) +{ + if (fork() == 0) + { + harnessTlsServerInit(TLS_TEST_PORT, TLS_CERT_TEST_CERT, TLS_CERT_TEST_KEY); + harnessTlsServerAccept(); + + // storageDriverS3NewRead() and StorageDriverS3FileRead + // ------------------------------------------------------------------------------------------------------------------------- + // Ignore missing file + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt")); + harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL)); + + // Error on missing file + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt")); + harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL)); + + // Get file + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt")); + harnessTlsServerReply(testS3ServerResponse(200, "OK", "this is a sample file")); + + // Throw non-404 error + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt")); + harnessTlsServerReply(testS3ServerResponse(303, "Some bad status", "CONTENT")); + + // storageDriverList() + // ------------------------------------------------------------------------------------------------------------------------- + // Throw error + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2")); + harnessTlsServerReply(testS3ServerResponse(344, "Another bad status", NULL)); + + // list a file/path in root + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2")); + harnessTlsServerReply( + testS3ServerResponse( + 200, "OK", + "" + "" + " " + " test1.txt" + " " + " " + " path1/" + " " + "")); + + // list a file in root with expression + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test")); + harnessTlsServerReply( + testS3ServerResponse( + 200, "OK", + "" + "" + " " + " test1.txt" + " " + "")); + + // list files with continuation + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F")); + harnessTlsServerReply( + testS3ServerResponse( + 200, "OK", + "" + "" + " 1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=" + " " + " path/to/test1.txt" + " " + " " + " path/to/test2.txt" + " " + " " + " path/to/path1/" + " " + "")); + + harnessTlsServerExpect( + testS3ServerRequest( + HTTP_VERB_GET, + "/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2" + "&prefix=path%2Fto%2F")); + harnessTlsServerReply( + testS3ServerResponse( + 200, "OK", + "" + "" + " " + " path/to/test3.txt" + " " + " " + " path/to/path2/" + " " + "")); + + // list files with expression + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest")); + harnessTlsServerReply( + testS3ServerResponse( + 200, "OK", + "" + "" + " " + " path/to/test1.txt" + " " + " " + " path/to/test2.txt" + " " + " " + " path/to/test3.txt" + " " + " " + " path/to/test1.path/" + " " + " " + " path/to/test2.path/" + " " + "")); + + harnessTlsServerClose(); + + exit(0); + } +} + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // Test strings + const String *path = strNew("/"); + const String *bucket = strNew("bucket"); + const String *region = strNew("us-east-1"); + const String *endPoint = strNew("s3.amazonaws.com"); + const String *host = strNew(TLS_TEST_HOST); + const unsigned int port = TLS_TEST_PORT; + const String *accessKey = strNew("AKIAIOSFODNN7EXAMPLE"); + const String *secretAccessKey = strNew("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + const String *securityToken = strNew( + "AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/q" + "kPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xV" + "qr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA=="); + + // ***************************************************************************************************************************** + if (testBegin("storageDriverS3New() and storageRepoGet()")) + { + // Only required options + // ------------------------------------------------------------------------------------------------------------------------- + StringList *argList = strLstNew(); + strLstAddZ(argList, "pgbackrest"); + strLstAddZ(argList, "--stanza=db"); + strLstAddZ(argList, "--repo1-type=s3"); + strLstAdd(argList, strNewFmt("--repo1-path=%s", strPtr(path))); + strLstAdd(argList, strNewFmt("--repo1-s3-bucket=%s", strPtr(bucket))); + strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region))); + strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s", strPtr(endPoint))); + setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); + setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); + strLstAddZ(argList, "archive-get"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + Storage *storage = NULL; + TEST_ASSIGN(storage, storageRepoGet(strNew(STORAGE_TYPE_S3), false), "get S3 repo storage"); + TEST_RESULT_STR(strPtr(storage->path), strPtr(path), " check path"); + TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->bucket), strPtr(bucket), " check bucket"); + TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->region), strPtr(region), " check region"); + TEST_RESULT_STR( + strPtr(((StorageDriverS3 *)storage->driver)->host), strPtr(strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint))), + " check host"); + TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->accessKey), strPtr(accessKey), " check access key"); + TEST_RESULT_STR( + strPtr(((StorageDriverS3 *)storage->driver)->secretAccessKey), strPtr(secretAccessKey), " check secret access key"); + TEST_RESULT_PTR(((StorageDriverS3 *)storage->driver)->securityToken, NULL, " check security token"); + + // Add default options + // ------------------------------------------------------------------------------------------------------------------------- + argList = strLstNew(); + strLstAddZ(argList, "pgbackrest"); + strLstAddZ(argList, "--stanza=db"); + strLstAddZ(argList, "--repo1-type=s3"); + strLstAdd(argList, strNewFmt("--repo1-path=%s", strPtr(path))); + strLstAdd(argList, strNewFmt("--repo1-s3-bucket=%s", strPtr(bucket))); + strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region))); + strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s", strPtr(endPoint))); + strLstAdd(argList, strNewFmt("--repo1-s3-host=%s", strPtr(host))); + strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); + strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_FAKE_PATH "/pgbackrest-test.crt"); + setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); + setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); + setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); + strLstAddZ(argList, "archive-get"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_ASSIGN(storage, storageRepoGet(strNew(STORAGE_TYPE_S3), false), "get S3 repo storage with options"); + TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->bucket), strPtr(bucket), " check bucket"); + TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->region), strPtr(region), " check region"); + TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->host), strPtr(host), " check host"); + TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->accessKey), strPtr(accessKey), " check access key"); + TEST_RESULT_STR( + strPtr(((StorageDriverS3 *)storage->driver)->secretAccessKey), strPtr(secretAccessKey), " check secret access key"); + TEST_RESULT_STR( + strPtr(((StorageDriverS3 *)storage->driver)->securityToken), strPtr(securityToken), " check security token"); + } + + // ***************************************************************************************************************************** + if (testBegin("storageDriverS3DateTime() and storageDriverS3Auth()")) + { + TEST_RESULT_STR(strPtr(storageDriverS3DateTime(1491267845)), "20170404T010405Z", "static date"); + + // ------------------------------------------------------------------------------------------------------------------------- + StorageDriverS3 *driver = storageDriverS3New( + path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, NULL, 0, 0, true, NULL, NULL); + + HttpHeader *header = httpHeaderNew(NULL); + + HttpQuery *query = httpQueryNew(); + httpQueryAdd(query, strNew("list-type"), strNew("2")); + + TEST_RESULT_VOID( + storageDriverS3Auth( + driver, strNew("GET"), strNew("/"), query, strNew("20170606T121212Z"), header, + strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")), + "generate authorization"); + TEST_RESULT_STR( + strPtr(httpHeaderGet(header, strNew("authorization"))), + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request," + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + "Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3", + " check authorization header"); + + // Test again to be sure cache signing key is used + const Buffer *lastSigningKey = driver->signingKey; + + TEST_RESULT_VOID( + storageDriverS3Auth( + driver, strNew("GET"), strNew("/"), query, strNew("20170606T121212Z"), header, + strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")), + "generate authorization"); + TEST_RESULT_STR( + strPtr(httpHeaderGet(header, strNew("authorization"))), + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request," + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + "Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3", + " check authorization header"); + TEST_RESULT_BOOL(driver->signingKey == lastSigningKey, true, " check signing key was reused"); + + // Change the date to generate a new signing key + TEST_RESULT_VOID( + storageDriverS3Auth( + driver, strNew("GET"), strNew("/"), query, strNew("20180814T080808Z"), header, + strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")), + " generate authorization"); + TEST_RESULT_STR( + strPtr(httpHeaderGet(header, strNew("authorization"))), + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20180814/us-east-1/s3/aws4_request," + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + "Signature=d0fa9c36426eb94cdbaf287a7872c7a3b6c913f523163d0d7debba0758e36f49", + " check authorization header"); + TEST_RESULT_BOOL(driver->signingKey != lastSigningKey, true, " check signing key was regenerated"); + + // Test with security token + // ------------------------------------------------------------------------------------------------------------------------- + driver = storageDriverS3New( + path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, securityToken, NULL, 0, 0, true, NULL, NULL); + + TEST_RESULT_VOID( + storageDriverS3Auth( + driver, strNew("GET"), strNew("/"), query, strNew("20170606T121212Z"), header, + strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")), + "generate authorization"); + TEST_RESULT_STR( + strPtr(httpHeaderGet(header, strNew("authorization"))), + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request," + "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token," + "Signature=c12565bf5d7e0ef623f76d66e09e5431aebef803f6a25a01c586525f17e474a3", + " check authorization header"); + } + + // ***************************************************************************************************************************** + if (testBegin("storageDriverS3*() and StorageDriverS3FileRead")) + { + testS3Server(); + + StorageDriverS3 *s3Driver = storageDriverS3New( + path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, host, port, 250, true, NULL, NULL); + Storage *s3 = storageDriverS3Interface(s3Driver); + + // Coverage for noop functions + // ------------------------------------------------------------------------------------------------------------------------- + TEST_RESULT_VOID(storagePathCreateNP(s3, strNew("path")), "path create is a noop"); + TEST_RESULT_VOID(storagePathSyncNP(s3, strNew("path")), "path sync is a noop"); + + // storageDriverS3NewRead() and StorageDriverS3FileRead + // ------------------------------------------------------------------------------------------------------------------------- + TEST_RESULT_PTR( + storageGetNP(storageNewReadP(s3, strNew("file.txt"), .ignoreMissing = true)), NULL, "ignore missing file"); + TEST_ERROR( + storageGetNP(storageNewReadNP(s3, strNew("file.txt"))), FileMissingError, + "unable to open '/file.txt': No such file or directory"); + TEST_RESULT_STR( + strPtr(strNewBuf(storageGetNP(storageNewReadNP(s3, strNew("file.txt"))))), "this is a sample file", + "get file"); + + StorageFileRead *read = NULL; + TEST_ASSIGN(read, storageNewReadP(s3, strNew("file.txt"), .ignoreMissing = true), "new read file"); + TEST_RESULT_BOOL(storageFileReadIgnoreMissing(read), true, " check ignore missing"); + TEST_RESULT_STR(strPtr(storageFileReadName(read)), "/file.txt", " check name"); + + TEST_ERROR( + ioReadOpen(storageFileReadIo(read)), ProtocolError, + "S3 request failed with 303: Some bad status\n" + "*** URI/Query ***:\n" + "/file.txt\n" + "*** Request Headers ***:\n" + "authorization: \n" + "content-length: 0\n" + "host: " TLS_TEST_HOST "\n" + "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" + "x-amz-date: \n" + "*** Response Headers ***:\n" + "content-length: 7\n" + "*** Response Content ***:\n" + "CONTENT") + + // storageDriverList() + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ERROR( + storageListP(s3, strNew("/"), .errorOnMissing = true), AssertError, "function test assertion '!errorOnMissing' failed"); + TEST_ERROR(storageListNP(s3, strNew("/")), ProtocolError, + "S3 request failed with 344: Another bad status\n" + "*** URI/Query ***:\n" + "/?delimiter=%2F&list-type=2\n" + "*** Request Headers ***:\n" + "authorization: \n" + "content-length: 0\n" + "host: " TLS_TEST_HOST "\n" + "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" + "x-amz-date: "); + + TEST_RESULT_STR(strPtr(strLstJoin(storageListNP(s3, strNew("/")), ",")), "path1,test1.txt", "list a file/path in root"); + TEST_RESULT_STR( + strPtr(strLstJoin(storageListP(s3, strNew("/"), .expression = strNew("^test.*$")), ",")), "test1.txt", + "list a file in root with expression"); + TEST_RESULT_STR( + strPtr(strLstJoin(storageListNP(s3, strNew("/path/to")), ",")), + "path1,test1.txt,test2.txt,path2,test3.txt", "list files with continuation"); + TEST_RESULT_STR( + strPtr(strLstJoin(storageListP(s3, strNew("/path/to"), .expression = strNew("^test(1|3)")), ",")), + "test1.path,test1.txt,test3.txt", "list files with expression"); + + // Coverage for unimplemented functions + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ERROR(storageExistsNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED"); + TEST_ERROR(storageInfoNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED"); + TEST_ERROR(storageNewWriteNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED"); + TEST_ERROR(storagePathRemoveNP(s3, strNew("path")), AssertError, "NOT YET IMPLEMENTED"); + TEST_ERROR(storageRemoveNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED"); + } + + FUNCTION_HARNESS_RESULT_VOID(); +}