mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-05-23 22:30:12 +02:00
Add S3 storage driver.
Only the storageNewRead() and storageList() functions are currently implemented, but this is enough to enable S3 for the archive-get command.
This commit is contained in:
David Steele
parent
72252ed2a1
commit
256b727a3d
doc/xml
src
test
@ -15,6 +15,10 @@
|
||||
<release date="XXXX-XX-XX" version="2.08dev" title="UNDER DEVELOPMENT">
|
||||
<release-core-list>
|
||||
<release-development-list>
|
||||
<release-item>
|
||||
<p>Add <proper>S3</proper> storage driver.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<p>Add <code>HttpClient</code> object.</p>
|
||||
</release-item>
|
||||
|
||||
10
src/Makefile
10
src/Makefile
@ -135,6 +135,8 @@ SRCS = \
|
||||
storage/driver/posix/common.c \
|
||||
storage/driver/posix/fileRead.c \
|
||||
storage/driver/posix/fileWrite.c \
|
||||
storage/driver/s3/fileRead.c \
|
||||
storage/driver/s3/storage.c \
|
||||
storage/fileRead.c \
|
||||
storage/fileWrite.c \
|
||||
storage/helper.c \
|
||||
@ -397,13 +399,19 @@ storage/driver/posix/fileWrite.o: storage/driver/posix/fileWrite.c common/assert
|
||||
storage/driver/posix/storage.o: storage/driver/posix/storage.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h storage/driver/posix/common.h storage/driver/posix/fileRead.h storage/driver/posix/fileWrite.h storage/driver/posix/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
|
||||
$(CC) $(CFLAGS) -c storage/driver/posix/storage.c -o storage/driver/posix/storage.o
|
||||
|
||||
storage/driver/s3/fileRead.o: storage/driver/s3/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h storage/driver/s3/fileRead.h storage/driver/s3/storage.h storage/fileRead.h storage/fileRead.intern.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
|
||||
$(CC) $(CFLAGS) -c storage/driver/s3/fileRead.c -o storage/driver/s3/fileRead.o
|
||||
|
||||
storage/driver/s3/storage.o: storage/driver/s3/storage.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/common.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/type/xml.h crypto/hash.h storage/driver/s3/fileRead.h storage/driver/s3/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
|
||||
$(CC) $(CFLAGS) -c storage/driver/s3/storage.c -o storage/driver/s3/storage.o
|
||||
|
||||
storage/fileRead.o: storage/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/fileRead.h storage/fileRead.intern.h
|
||||
$(CC) $(CFLAGS) -c storage/fileRead.c -o storage/fileRead.o
|
||||
|
||||
storage/fileWrite.o: storage/fileWrite.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/fileWrite.h storage/fileWrite.intern.h version.h
|
||||
$(CC) $(CFLAGS) -c storage/fileWrite.c -o storage/fileWrite.o
|
||||
|
||||
storage/helper.o: storage/helper.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/driver/posix/fileRead.h storage/driver/posix/fileWrite.h storage/driver/posix/storage.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h storage/storage.intern.h
|
||||
storage/helper.o: storage/helper.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/driver/posix/fileRead.h storage/driver/posix/fileWrite.h storage/driver/posix/storage.h storage/driver/s3/storage.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h storage/storage.intern.h
|
||||
$(CC) $(CFLAGS) -c storage/helper.c -o storage/helper.o
|
||||
|
||||
storage/storage.o: storage/storage.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
|
||||
|
||||
190
src/storage/driver/s3/fileRead.c
Normal file
190
src/storage/driver/s3/fileRead.c
Normal file
@ -0,0 +1,190 @@
|
||||
/***********************************************************************************************************************************
|
||||
S3 Storage File Read Driver
|
||||
***********************************************************************************************************************************/
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/io/http/client.h"
|
||||
#include "common/io/read.intern.h"
|
||||
#include "common/log.h"
|
||||
#include "common/memContext.h"
|
||||
#include "storage/driver/s3/fileRead.h"
|
||||
#include "storage/fileRead.intern.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
***********************************************************************************************************************************/
|
||||
struct StorageDriverS3FileRead
|
||||
{
|
||||
MemContext *memContext;
|
||||
StorageDriverS3 *storage;
|
||||
StorageFileRead *interface;
|
||||
IoRead *io;
|
||||
String *name;
|
||||
bool ignoreMissing;
|
||||
|
||||
HttpClient *httpClient; // Http client for requests
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Create a new file
|
||||
***********************************************************************************************************************************/
|
||||
StorageDriverS3FileRead *storageDriverS3FileReadNew(StorageDriverS3 *storage, const String *name, bool ignoreMissing)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
||||
FUNCTION_DEBUG_PARAM(STRING, name);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing);
|
||||
|
||||
FUNCTION_TEST_ASSERT(name != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
StorageDriverS3FileRead *this = NULL;
|
||||
|
||||
// Create the file object
|
||||
MEM_CONTEXT_NEW_BEGIN("StorageDriverS3FileRead")
|
||||
{
|
||||
this = memNew(sizeof(StorageDriverS3FileRead));
|
||||
this->memContext = MEM_CONTEXT_NEW();
|
||||
this->storage = storage;
|
||||
this->name = strDup(name);
|
||||
this->ignoreMissing = ignoreMissing;
|
||||
|
||||
this->interface = storageFileReadNewP(
|
||||
strNew(STORAGE_DRIVER_S3_TYPE), this,
|
||||
.ignoreMissing = (StorageFileReadInterfaceIgnoreMissing)storageDriverS3FileReadIgnoreMissing,
|
||||
.io = (StorageFileReadInterfaceIo)storageDriverS3FileReadIo,
|
||||
.name = (StorageFileReadInterfaceName)storageDriverS3FileReadName);
|
||||
|
||||
this->io = ioReadNewP(
|
||||
this, .eof = (IoReadInterfaceEof)storageDriverS3FileReadEof,
|
||||
.open = (IoReadInterfaceOpen)storageDriverS3FileReadOpen, .read = (IoReadInterfaceRead)storageDriverS3FileRead);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT(STORAGE_DRIVER_S3_FILE_READ, this);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Open the file
|
||||
***********************************************************************************************************************************/
|
||||
bool
|
||||
storageDriverS3FileReadOpen(StorageDriverS3FileRead *this)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3_FILE_READ, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_ASSERT(this->httpClient == NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
bool result = false;
|
||||
|
||||
// Request the file
|
||||
storageDriverS3Request(this->storage, HTTP_VERB_GET_STR, this->name, NULL, NULL, false, true);
|
||||
|
||||
// On success
|
||||
this->httpClient = storageDriverS3HttpClient(this->storage);
|
||||
|
||||
if (httpClientResponseCode(this->httpClient) == HTTP_RESPONSE_CODE_OK)
|
||||
result = true;
|
||||
|
||||
// Else error unless ignore missing
|
||||
else if (!this->ignoreMissing)
|
||||
THROW_FMT(FileMissingError, "unable to open '%s': No such file or directory", strPtr(this->name));
|
||||
|
||||
FUNCTION_DEBUG_RESULT(BOOL, result);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Read from a file
|
||||
***********************************************************************************************************************************/
|
||||
size_t
|
||||
storageDriverS3FileRead(StorageDriverS3FileRead *this, Buffer *buffer)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3_FILE_READ, this);
|
||||
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
|
||||
|
||||
FUNCTION_DEBUG_ASSERT(this != NULL && this->httpClient != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(buffer != NULL && !bufFull(buffer));
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT(SIZE, ioRead(httpClientIoRead(this->httpClient), buffer));
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Has file reached EOF?
|
||||
***********************************************************************************************************************************/
|
||||
bool
|
||||
storageDriverS3FileReadEof(const StorageDriverS3FileRead *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this);
|
||||
|
||||
FUNCTION_DEBUG_ASSERT(this != NULL && this->httpClient != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(BOOL, ioReadEof(httpClientIoRead(this->httpClient)));
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Should a missing file be ignored?
|
||||
***********************************************************************************************************************************/
|
||||
bool
|
||||
storageDriverS3FileReadIgnoreMissing(const StorageDriverS3FileRead *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(BOOL, this->ignoreMissing);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get the interface
|
||||
***********************************************************************************************************************************/
|
||||
StorageFileRead *
|
||||
storageDriverS3FileReadInterface(const StorageDriverS3FileRead *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(STORAGE_FILE_READ, this->interface);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get the I/O interface
|
||||
***********************************************************************************************************************************/
|
||||
IoRead *
|
||||
storageDriverS3FileReadIo(const StorageDriverS3FileRead *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(IO_READ, this->io);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
File name
|
||||
***********************************************************************************************************************************/
|
||||
const String *
|
||||
storageDriverS3FileReadName(const StorageDriverS3FileRead *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_READ, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(CONST_STRING, this->name);
|
||||
}
|
||||
50
src/storage/driver/s3/fileRead.h
Normal file
50
src/storage/driver/s3/fileRead.h
Normal file
@ -0,0 +1,50 @@
|
||||
/***********************************************************************************************************************************
|
||||
S3 Storage File Read Driver
|
||||
***********************************************************************************************************************************/
|
||||
#ifndef STORAGE_DRIVER_S3_FILEREAD_H
|
||||
#define STORAGE_DRIVER_S3_FILEREAD_H
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct StorageDriverS3FileRead StorageDriverS3FileRead;
|
||||
|
||||
#include "common/type/buffer.h"
|
||||
#include "common/type/string.h"
|
||||
#include "storage/driver/s3/storage.h"
|
||||
#include "storage/fileRead.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructor
|
||||
***********************************************************************************************************************************/
|
||||
StorageDriverS3FileRead *storageDriverS3FileReadNew(StorageDriverS3 *storage, const String *name, bool ignoreMissing);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
bool storageDriverS3FileReadOpen(StorageDriverS3FileRead *this);
|
||||
size_t storageDriverS3FileRead(StorageDriverS3FileRead *this, Buffer *buffer);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Getters
|
||||
***********************************************************************************************************************************/
|
||||
bool storageDriverS3FileReadEof(const StorageDriverS3FileRead *this);
|
||||
bool storageDriverS3FileReadIgnoreMissing(const StorageDriverS3FileRead *this);
|
||||
StorageFileRead *storageDriverS3FileReadInterface(const StorageDriverS3FileRead *this);
|
||||
IoRead *storageDriverS3FileReadIo(const StorageDriverS3FileRead *this);
|
||||
const String *storageDriverS3FileReadName(const StorageDriverS3FileRead *this);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Destructor
|
||||
***********************************************************************************************************************************/
|
||||
void storageDriverS3FileReadFree(StorageDriverS3FileRead *this);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Macros for function logging
|
||||
***********************************************************************************************************************************/
|
||||
#define FUNCTION_DEBUG_STORAGE_DRIVER_S3_FILE_READ_TYPE \
|
||||
StorageDriverS3FileRead *
|
||||
#define FUNCTION_DEBUG_STORAGE_DRIVER_S3_FILE_READ_FORMAT(value, buffer, bufferSize) \
|
||||
objToLog(value, "StorageDriverS3FileRead", buffer, bufferSize)
|
||||
|
||||
#endif
|
||||
702
src/storage/driver/s3/storage.c
Normal file
702
src/storage/driver/s3/storage.c
Normal file
@ -0,0 +1,702 @@
|
||||
/***********************************************************************************************************************************
|
||||
S3 Storage Driver
|
||||
***********************************************************************************************************************************/
|
||||
#include <time.h>
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/io/http/common.h"
|
||||
#include "common/log.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/regExp.h"
|
||||
#include "common/type/xml.h"
|
||||
#include "crypto/hash.h"
|
||||
#include "storage/driver/s3/fileRead.h"
|
||||
#include "storage/driver/s3/storage.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Driver type constant string
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(STORAGE_DRIVER_S3_TYPE_STR, STORAGE_DRIVER_S3_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
S3 http headers
|
||||
***********************************************************************************************************************************/
|
||||
STRING_STATIC(S3_HEADER_AUTHORIZATION_STR, "authorization");
|
||||
STRING_STATIC(S3_HEADER_CONTENT_SHA256_STR, "x-amz-content-sha256");
|
||||
STRING_STATIC(S3_HEADER_DATE_STR, "x-amz-date");
|
||||
STRING_STATIC(S3_HEADER_HOST_STR, "host");
|
||||
STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-security-token");
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
S3 query tokens
|
||||
***********************************************************************************************************************************/
|
||||
STRING_STATIC(S3_QUERY_CONTINUATION_TOKEN_STR, "continuation-token");
|
||||
STRING_STATIC(S3_QUERY_DELIMITER_STR, "delimiter");
|
||||
STRING_STATIC(S3_QUERY_LIST_TYPE_STR, "list-type");
|
||||
STRING_STATIC(S3_QUERY_PREFIX_STR, "prefix");
|
||||
|
||||
STRING_STATIC(S3_QUERY_VALUE_LIST_TYPE_2_STR, "2");
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
XML tags
|
||||
***********************************************************************************************************************************/
|
||||
STRING_STATIC(S3_XML_TAG_COMMON_PREFIXES_STR, "CommonPrefixes");
|
||||
STRING_STATIC(S3_XML_TAG_CONTENTS_STR, "Contents");
|
||||
STRING_STATIC(S3_XML_TAG_KEY_STR, "Key");
|
||||
STRING_STATIC(S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, "NextContinuationToken");
|
||||
STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix");
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
AWS authentication v4 constants
|
||||
***********************************************************************************************************************************/
|
||||
#define S3 "s3"
|
||||
#define AWS4 "AWS4"
|
||||
#define AWS4_REQUEST "aws4_request"
|
||||
#define AWS4_HMAC_SHA256 "AWS4-HMAC-SHA256"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Starting data for signing string so it will be regenerated on the first request
|
||||
***********************************************************************************************************************************/
|
||||
STRING_STATIC(YYYYMMDD_STR, "YYYYMMDD");
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
***********************************************************************************************************************************/
|
||||
struct StorageDriverS3
|
||||
{
|
||||
MemContext *memContext;
|
||||
Storage *interface; // Driver interface
|
||||
HttpClient *httpClient; // Http client to service requests
|
||||
const StringList *headerRedactList; // List of headers to redact from logging
|
||||
|
||||
const String *bucket; // Bucket to store data in
|
||||
const String *region; // e.g. us-east-1
|
||||
const String *accessKey; // Access key
|
||||
const String *secretAccessKey; // Secret access key
|
||||
const String *securityToken; // Security token, if any
|
||||
const String *host; // Defaults to {bucket}.{endpoint}
|
||||
|
||||
// Current signing key and date it is valid for
|
||||
const String *signingKeyDate; // Date of cached signing key (so we know when to regenerate)
|
||||
const Buffer *signingKey; // Cached signing key
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Expected ISO-8601 data/time size
|
||||
***********************************************************************************************************************************/
|
||||
#define ISO_8601_DATE_TIME_SIZE 16
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Format ISO-8601 date/time for authentication
|
||||
***********************************************************************************************************************************/
|
||||
static String *
|
||||
storageDriverS3DateTime(time_t authTime)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(INT64, authTime);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
char buffer[ISO_8601_DATE_TIME_SIZE + 1];
|
||||
|
||||
if (strftime( // {uncoverable - nothing invalid can be passed}
|
||||
buffer, sizeof(buffer), "%Y%m%dT%H%M%SZ", gmtime(&authTime)) != ISO_8601_DATE_TIME_SIZE)
|
||||
THROW_SYS_ERROR(AssertError, "unable to format date"); // {+uncoverable}
|
||||
|
||||
FUNCTION_TEST_RESULT(STRING, strNew(buffer));
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Generate authorization header and add it to the supplied header list
|
||||
|
||||
Based on the excellent documentation at http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html.
|
||||
***********************************************************************************************************************************/
|
||||
static void
|
||||
storageDriverS3Auth(
|
||||
StorageDriverS3 *this, const String *verb, const String *uri, const HttpQuery *query, const String *dateTime,
|
||||
HttpHeader *httpHeader, const String *payloadHash)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_TEST_PARAM(STRING, verb);
|
||||
FUNCTION_TEST_PARAM(STRING, uri);
|
||||
FUNCTION_TEST_PARAM(HTTP_QUERY, query);
|
||||
FUNCTION_TEST_PARAM(STRING, dateTime);
|
||||
FUNCTION_TEST_PARAM(KEY_VALUE, httpHeader);
|
||||
FUNCTION_TEST_PARAM(STRING, payloadHash);
|
||||
|
||||
FUNCTION_TEST_ASSERT(verb != NULL);
|
||||
FUNCTION_TEST_ASSERT(uri != NULL);
|
||||
FUNCTION_TEST_ASSERT(dateTime != NULL);
|
||||
FUNCTION_TEST_ASSERT(httpHeader != NULL);
|
||||
FUNCTION_TEST_ASSERT(payloadHash != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
// Get date from datetime
|
||||
const String *date = strSubN(dateTime, 0, 8);
|
||||
|
||||
// Set required headers
|
||||
httpHeaderPut(httpHeader, S3_HEADER_CONTENT_SHA256_STR, payloadHash);
|
||||
httpHeaderPut(httpHeader, S3_HEADER_DATE_STR, dateTime);
|
||||
httpHeaderPut(httpHeader, S3_HEADER_HOST_STR, this->host);
|
||||
|
||||
if (this->securityToken != NULL)
|
||||
httpHeaderPut(httpHeader, S3_HEADER_TOKEN_STR, this->securityToken);
|
||||
|
||||
// Generate canonical request and signed headers
|
||||
const StringList *headerList = strLstSort(strLstDup(httpHeaderList(httpHeader)), sortOrderAsc);
|
||||
String *signedHeaders = NULL;
|
||||
|
||||
String *canonicalRequest = strNewFmt(
|
||||
"%s\n%s\n%s\n", strPtr(verb), strPtr(uri), query == NULL ? "" : strPtr(httpQueryRender(query)));
|
||||
|
||||
for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
|
||||
{
|
||||
const String *headerKey = strLstGet(headerList, headerIdx);
|
||||
const String *headerKeyLower = strLower(strDup(headerKey));
|
||||
|
||||
// Skip the authorization header -- if it exists this is a retry
|
||||
if (strEq(headerKeyLower, S3_HEADER_AUTHORIZATION_STR))
|
||||
continue;
|
||||
|
||||
strCatFmt(canonicalRequest, "%s:%s\n", strPtr(headerKeyLower), strPtr(httpHeaderGet(httpHeader, headerKey)));
|
||||
|
||||
if (signedHeaders == NULL)
|
||||
signedHeaders = strDup(headerKeyLower);
|
||||
else
|
||||
strCatFmt(signedHeaders, ";%s", strPtr(headerKeyLower));
|
||||
}
|
||||
|
||||
strCatFmt(canonicalRequest, "\n%s\n%s", strPtr(signedHeaders), strPtr(payloadHash));
|
||||
|
||||
// Generate string to sign
|
||||
const String *stringToSign = strNewFmt(
|
||||
AWS4_HMAC_SHA256 "\n%s\n%s/%s/" S3 "/" AWS4_REQUEST "\n%s", strPtr(dateTime), strPtr(date), strPtr(this->region),
|
||||
strPtr(bufHex(cryptoHashOneStr(HASH_TYPE_SHA256_STR, canonicalRequest))));
|
||||
|
||||
// Generate signing key. This key only needs to be regenerated every seven days but we'll do it once a day to keep the
|
||||
// logic simple. It's a relatively expensive operation so we'd rather not do it for every request.
|
||||
// If the cached signing key has expired (or has noe been generated) then regenerate it
|
||||
if (!strEq(date, this->signingKeyDate))
|
||||
{
|
||||
const Buffer *dateKey = cryptoHmacOne(
|
||||
HASH_TYPE_SHA256_STR, bufNewStr(strNewFmt(AWS4 "%s", strPtr(this->secretAccessKey))), bufNewStr(date));
|
||||
const Buffer *regionKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, dateKey, bufNewStr(this->region));
|
||||
const Buffer *serviceKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, regionKey, bufNewZ(S3));
|
||||
|
||||
// Switch to the object context so signing key and date are not lost
|
||||
MEM_CONTEXT_BEGIN(this->memContext)
|
||||
{
|
||||
this->signingKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, serviceKey, bufNewZ(AWS4_REQUEST));
|
||||
this->signingKeyDate = strDup(date);
|
||||
}
|
||||
MEM_CONTEXT_END();
|
||||
}
|
||||
|
||||
// Generate authorization header
|
||||
const String *authorization = strNewFmt(
|
||||
AWS4_HMAC_SHA256 " Credential=%s/%s/%s/" S3 "/" AWS4_REQUEST ",SignedHeaders=%s,Signature=%s",
|
||||
strPtr(this->accessKey), strPtr(date), strPtr(this->region), strPtr(signedHeaders),
|
||||
strPtr(bufHex(cryptoHmacOne(HASH_TYPE_SHA256_STR, this->signingKey, bufNewStr(stringToSign)))));
|
||||
|
||||
httpHeaderPut(httpHeader, S3_HEADER_AUTHORIZATION_STR, authorization);
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_TEST_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
New object
|
||||
***********************************************************************************************************************************/
|
||||
StorageDriverS3 *
|
||||
storageDriverS3New(
|
||||
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
|
||||
const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey,
|
||||
const String *securityToken, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile,
|
||||
const String *caPath)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STRING, bucket);
|
||||
FUNCTION_DEBUG_PARAM(STRING, endPoint);
|
||||
FUNCTION_DEBUG_PARAM(STRING, region);
|
||||
FUNCTION_TEST_PARAM(STRING, accessKey);
|
||||
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
|
||||
FUNCTION_TEST_PARAM(STRING, securityToken);
|
||||
FUNCTION_DEBUG_PARAM(STRING, host);
|
||||
FUNCTION_DEBUG_PARAM(UINT, port);
|
||||
FUNCTION_DEBUG_PARAM(TIME_MSEC, timeout);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, verifyPeer);
|
||||
FUNCTION_DEBUG_PARAM(STRING, caFile);
|
||||
FUNCTION_DEBUG_PARAM(STRING, caPath);
|
||||
|
||||
FUNCTION_TEST_ASSERT(bucket != NULL);
|
||||
FUNCTION_TEST_ASSERT(endPoint != NULL);
|
||||
FUNCTION_TEST_ASSERT(region != NULL);
|
||||
FUNCTION_TEST_ASSERT(accessKey != NULL);
|
||||
FUNCTION_TEST_ASSERT(secretAccessKey != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
// Create the object
|
||||
StorageDriverS3 *this = NULL;
|
||||
|
||||
MEM_CONTEXT_NEW_BEGIN("StorageDriverS3")
|
||||
{
|
||||
this = memNew(sizeof(StorageDriverS3));
|
||||
this->memContext = MEM_CONTEXT_NEW();
|
||||
|
||||
this->bucket = strDup(bucket);
|
||||
this->region = strDup(region);
|
||||
this->accessKey = strDup(accessKey);
|
||||
this->secretAccessKey = strDup(secretAccessKey);
|
||||
this->securityToken = strDup(securityToken);
|
||||
this->host = host == NULL ? strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint)) : strDup(host);
|
||||
|
||||
// Force the signing key to be generated on the first run
|
||||
this->signingKeyDate = YYYYMMDD_STR;
|
||||
|
||||
// Create the storage interface
|
||||
this->interface = storageNewP(
|
||||
STORAGE_DRIVER_S3_TYPE_STR, path, 0, 0, write, pathExpressionFunction, this,
|
||||
.exists = (StorageInterfaceExists)storageDriverS3Exists, .info = (StorageInterfaceInfo)storageDriverS3Info,
|
||||
.list = (StorageInterfaceList)storageDriverS3List, .newRead = (StorageInterfaceNewRead)storageDriverS3NewRead,
|
||||
.newWrite = (StorageInterfaceNewWrite)storageDriverS3NewWrite,
|
||||
.pathCreate = (StorageInterfacePathCreate)storageDriverS3PathCreate,
|
||||
.pathRemove = (StorageInterfacePathRemove)storageDriverS3PathRemove,
|
||||
.pathSync = (StorageInterfacePathSync)storageDriverS3PathSync, .remove = (StorageInterfaceRemove)storageDriverS3Remove);
|
||||
|
||||
// Create the http client used to service requests
|
||||
this->httpClient = httpClientNew(this->host, port, timeout, verifyPeer, caFile, caPath);
|
||||
this->headerRedactList = strLstAdd(strLstNew(), S3_HEADER_AUTHORIZATION_STR);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT(STORAGE_DRIVER_S3, this);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Process S3 request
|
||||
***********************************************************************************************************************************/
|
||||
Buffer *
|
||||
storageDriverS3Request(
|
||||
StorageDriverS3 *this, const String *verb, const String *uri, const HttpQuery *query, const Buffer *body, bool returnContent,
|
||||
bool allowMissing)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, verb);
|
||||
FUNCTION_DEBUG_PARAM(STRING, uri);
|
||||
FUNCTION_DEBUG_PARAM(HTTP_QUERY, query);
|
||||
FUNCTION_DEBUG_PARAM(BUFFER, body);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, returnContent);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, allowMissing);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_ASSERT(verb != NULL);
|
||||
FUNCTION_TEST_ASSERT(uri != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
Buffer *result = NULL;
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
// Create header list and add content length
|
||||
HttpHeader *requestHeader = httpHeaderNew(this->headerRedactList);
|
||||
httpHeaderAdd(requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
|
||||
|
||||
// Generate authorization header
|
||||
storageDriverS3Auth(
|
||||
this, verb, uri, query, storageDriverS3DateTime(time(NULL)), requestHeader,
|
||||
strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"));
|
||||
|
||||
// Process request
|
||||
result = httpClientRequest(this->httpClient, verb, uri, query, requestHeader, returnContent);
|
||||
|
||||
// Error if the request was not successful
|
||||
if (httpClientResponseCode(this->httpClient) != HTTP_RESPONSE_CODE_OK &&
|
||||
(!allowMissing || httpClientResponseCode(this->httpClient) != HTTP_RESPONSE_CODE_NOT_FOUND))
|
||||
{
|
||||
// General error message
|
||||
String *error = strNewFmt(
|
||||
"S3 request failed with %u: %s", httpClientResponseCode(this->httpClient),
|
||||
strPtr(httpClientResponseMessage(this->httpClient)));
|
||||
|
||||
// Output uri/query
|
||||
strCat(error, "\n*** URI/Query ***:");
|
||||
|
||||
strCatFmt(error, "\n%s", strPtr(httpUriEncode(uri, true)));
|
||||
|
||||
if (query != NULL)
|
||||
strCatFmt(error, "?%s", strPtr(httpQueryRender(query)));
|
||||
|
||||
// Output request headers
|
||||
const StringList *requestHeaderList = httpHeaderList(requestHeader);
|
||||
|
||||
strCat(error, "\n*** Request Headers ***:");
|
||||
|
||||
for (unsigned int requestHeaderIdx = 0; requestHeaderIdx < strLstSize(requestHeaderList); requestHeaderIdx++)
|
||||
{
|
||||
const String *key = strLstGet(requestHeaderList, requestHeaderIdx);
|
||||
|
||||
strCatFmt(
|
||||
error, "\n%s: %s", strPtr(key),
|
||||
httpHeaderRedact(requestHeader, key) || strEq(key, S3_HEADER_DATE_STR) ?
|
||||
"<redacted>" : strPtr(httpHeaderGet(requestHeader, key)));
|
||||
}
|
||||
|
||||
// Output response headers
|
||||
const HttpHeader *responseHeader = httpClientReponseHeader(this->httpClient);
|
||||
const StringList *responseHeaderList = httpHeaderList(responseHeader);
|
||||
|
||||
if (strLstSize(responseHeaderList) > 0)
|
||||
{
|
||||
strCat(error, "\n*** Response Headers ***:");
|
||||
|
||||
for (unsigned int responseHeaderIdx = 0; responseHeaderIdx < strLstSize(responseHeaderList); responseHeaderIdx++)
|
||||
{
|
||||
const String *key = strLstGet(responseHeaderList, responseHeaderIdx);
|
||||
strCatFmt(error, "\n%s: %s", strPtr(key), strPtr(httpHeaderGet(responseHeader, key)));
|
||||
}
|
||||
}
|
||||
|
||||
// If there was content then output it
|
||||
if (result != NULL)
|
||||
strCatFmt(error, "\n*** Response Content ***:\n%s", strPtr(strNewBuf(result)));
|
||||
|
||||
THROW(ProtocolError, strPtr(error));
|
||||
}
|
||||
|
||||
// On success move the buffer to the calling context
|
||||
bufMove(result, MEM_CONTEXT_OLD());
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT(BUFFER, result);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Does a file/path exist?
|
||||
***********************************************************************************************************************************/
|
||||
bool
|
||||
storageDriverS3Exists(StorageDriverS3 *this, const String *path)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, path);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(path != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
bool result = false;
|
||||
|
||||
THROW(AssertError, "NOT YET IMPLEMENTED");
|
||||
|
||||
FUNCTION_DEBUG_RESULT(BOOL, result);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
File/path info
|
||||
***********************************************************************************************************************************/
|
||||
StorageInfo
|
||||
storageDriverS3Info(StorageDriverS3 *this, const String *file, bool ignoreMissing)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, file);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(file != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
THROW(AssertError, "NOT YET IMPLEMENTED");
|
||||
|
||||
FUNCTION_DEBUG_RESULT(STORAGE_INFO, (StorageInfo){0});
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get a list of files from a directory
|
||||
***********************************************************************************************************************************/
|
||||
StringList *
|
||||
storageDriverS3List(StorageDriverS3 *this, const String *path, bool errorOnMissing, const String *expression)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, path);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, errorOnMissing);
|
||||
FUNCTION_DEBUG_PARAM(STRING, expression);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(path != NULL);
|
||||
FUNCTION_TEST_ASSERT(!errorOnMissing);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
StringList *result = NULL;
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
result = strLstNew();
|
||||
const String *continuationToken = NULL;
|
||||
|
||||
// Prepare regexp if an expression was passed
|
||||
RegExp *regExp = (expression == NULL) ? NULL : regExpNew(expression);
|
||||
|
||||
// Build the base prefix by stripping of the initial /
|
||||
const String *basePrefix;
|
||||
|
||||
if (strSize(path) == 1)
|
||||
basePrefix = EMPTY_STR;
|
||||
else
|
||||
basePrefix = strNewFmt("%s/", strPtr(strSub(path, 1)));
|
||||
|
||||
// Get the expression prefix when possible to limit initial results
|
||||
const String *expressionPrefix = regExpPrefix(expression);
|
||||
|
||||
// If there is an expression prefix then use it to build the query prefix, otherwise query prefix is base prefix
|
||||
const String *queryPrefix;
|
||||
|
||||
if (expressionPrefix == NULL)
|
||||
queryPrefix = basePrefix;
|
||||
else
|
||||
{
|
||||
if (strEmpty(basePrefix))
|
||||
queryPrefix = expressionPrefix;
|
||||
else
|
||||
queryPrefix = strNewFmt("%s%s", strPtr(basePrefix), strPtr(expressionPrefix));
|
||||
}
|
||||
|
||||
// Loop as long as a continuation token returned
|
||||
do
|
||||
{
|
||||
// Use an inner mem context here because we could potentially be retrieving millions of files so it is a good idea to
|
||||
// free memory at regular intervals
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
HttpQuery *query = httpQueryNew();
|
||||
|
||||
// Add continuation token from the prior loop if any
|
||||
if (continuationToken != NULL)
|
||||
httpQueryAdd(query, S3_QUERY_CONTINUATION_TOKEN_STR, continuationToken);
|
||||
|
||||
// Add the delimiter so we don't recurse
|
||||
httpQueryAdd(query, S3_QUERY_DELIMITER_STR, FSLASH_STR);
|
||||
|
||||
// Use list type 2
|
||||
httpQueryAdd(query, S3_QUERY_LIST_TYPE_STR, S3_QUERY_VALUE_LIST_TYPE_2_STR);
|
||||
|
||||
// Don't specified empty prefix because it is the default
|
||||
if (!strEmpty(queryPrefix))
|
||||
httpQueryAdd(query, S3_QUERY_PREFIX_STR, queryPrefix);
|
||||
|
||||
XmlNode *xmlRoot = xmlDocumentRoot(
|
||||
xmlDocumentNewBuf(storageDriverS3Request(this, HTTP_VERB_GET_STR, FSLASH_STR, query, NULL, true, false)));
|
||||
|
||||
// Get subpath list
|
||||
XmlNodeList *subPathList = xmlNodeChildList(xmlRoot, S3_XML_TAG_COMMON_PREFIXES_STR);
|
||||
|
||||
for (unsigned int subPathIdx = 0; subPathIdx < xmlNodeLstSize(subPathList); subPathIdx++)
|
||||
{
|
||||
// Get subpath name
|
||||
const String *subPath = xmlNodeContent(
|
||||
xmlNodeChild(xmlNodeLstGet(subPathList, subPathIdx), S3_XML_TAG_PREFIX_STR, true));
|
||||
|
||||
// Strip off base prefix and final /
|
||||
subPath = strSubN(subPath, strSize(basePrefix), strSize(subPath) - strSize(basePrefix) - 1);
|
||||
|
||||
// Add to list after checking expression if present
|
||||
if (regExp == NULL || regExpMatch(regExp, subPath))
|
||||
strLstAdd(result, subPath);
|
||||
}
|
||||
|
||||
// Get file list
|
||||
XmlNodeList *fileList = xmlNodeChildList(xmlRoot, S3_XML_TAG_CONTENTS_STR);
|
||||
|
||||
for (unsigned int fileIdx = 0; fileIdx < xmlNodeLstSize(fileList); fileIdx++)
|
||||
{
|
||||
// Get file name
|
||||
const String *file = xmlNodeContent(xmlNodeChild(xmlNodeLstGet(fileList, fileIdx), S3_XML_TAG_KEY_STR, true));
|
||||
|
||||
// Strip off the base prefix when present
|
||||
file = strEmpty(basePrefix) ? file : strSub(file, strSize(basePrefix));
|
||||
|
||||
// Add to list after checking expression if present
|
||||
if (regExp == NULL || regExpMatch(regExp, file))
|
||||
strLstAdd(result, file);
|
||||
}
|
||||
|
||||
// Get the continuation token and store it in the outer temp context
|
||||
memContextSwitch(MEM_CONTEXT_OLD());
|
||||
continuationToken = xmlNodeContent(xmlNodeChild(xmlRoot, S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, false));
|
||||
memContextSwitch(MEM_CONTEXT_TEMP());
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
}
|
||||
while (continuationToken != NULL);
|
||||
|
||||
strLstMove(result, MEM_CONTEXT_OLD());
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT(STRING_LIST, result);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
New file read object
|
||||
***********************************************************************************************************************************/
|
||||
StorageFileRead *
|
||||
storageDriverS3NewRead(StorageDriverS3 *this, const String *file, bool ignoreMissing)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, file);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(file != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT(
|
||||
STORAGE_FILE_READ, storageDriverS3FileReadInterface(storageDriverS3FileReadNew(this, file, ignoreMissing)));
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
New file write object
|
||||
***********************************************************************************************************************************/
|
||||
StorageFileWrite *
|
||||
storageDriverS3NewWrite(
|
||||
StorageDriverS3 *this, const String *file, mode_t modeFile, mode_t modePath, bool createPath, bool syncFile,
|
||||
bool syncPath, bool atomic)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, file);
|
||||
FUNCTION_DEBUG_PARAM(MODE, modeFile);
|
||||
FUNCTION_DEBUG_PARAM(MODE, modePath);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, createPath);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, syncFile);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, syncPath);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, atomic);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(file != NULL);
|
||||
FUNCTION_TEST_ASSERT(modeFile == 0);
|
||||
FUNCTION_TEST_ASSERT(modePath == 0);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
THROW(AssertError, "NOT YET IMPLEMENTED");
|
||||
|
||||
FUNCTION_DEBUG_RESULT(STORAGE_FILE_WRITE, NULL);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Create a path. There are no physical paths on S3 so just return success.
|
||||
***********************************************************************************************************************************/
|
||||
void
|
||||
storageDriverS3PathCreate(StorageDriverS3 *this, const String *path, bool errorOnExists, bool noParentCreate, mode_t mode)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, path);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, errorOnExists);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, noParentCreate);
|
||||
FUNCTION_DEBUG_PARAM(MODE, mode);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(path != NULL);
|
||||
FUNCTION_TEST_ASSERT(mode == 0);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Remove a path
|
||||
***********************************************************************************************************************************/
|
||||
void
|
||||
storageDriverS3PathRemove(StorageDriverS3 *this, const String *path, bool errorOnMissing, bool recurse)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, path);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, errorOnMissing);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, recurse);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(path != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
THROW(AssertError, "NOT YET IMPLEMENTED");
|
||||
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Sync a path. There's no need for this on S3 so just return success.
|
||||
***********************************************************************************************************************************/
|
||||
void
|
||||
storageDriverS3PathSync(StorageDriverS3 *this, const String *path, bool ignoreMissing)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, path);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(path != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Remove a file
|
||||
***********************************************************************************************************************************/
|
||||
void
|
||||
storageDriverS3Remove(StorageDriverS3 *this, const String *file, bool errorOnMissing)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
FUNCTION_DEBUG_PARAM(STRING, file);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, errorOnMissing);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(file != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
THROW(AssertError, "NOT YET IMPLEMENTED");
|
||||
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get http client
|
||||
***********************************************************************************************************************************/
|
||||
HttpClient *
|
||||
storageDriverS3HttpClient(const StorageDriverS3 *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(HTTP_CLIENT, this->httpClient);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get storage interface
|
||||
***********************************************************************************************************************************/
|
||||
Storage *
|
||||
storageDriverS3Interface(const StorageDriverS3 *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_DRIVER_S3, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(STORAGE, this->interface);
|
||||
}
|
||||
75
src/storage/driver/s3/storage.h
Normal file
75
src/storage/driver/s3/storage.h
Normal file
@ -0,0 +1,75 @@
|
||||
/***********************************************************************************************************************************
|
||||
S3 Storage Driver
|
||||
***********************************************************************************************************************************/
|
||||
#ifndef STORAGE_DRIVER_S3_STORAGE_H
|
||||
#define STORAGE_DRIVER_S3_STORAGE_H
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct StorageDriverS3 StorageDriverS3;
|
||||
|
||||
#include "common/io/http/client.h"
|
||||
#include "common/type/string.h"
|
||||
#include "storage/storage.intern.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Driver type constant
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_DRIVER_S3_TYPE "s3"
|
||||
STRING_DECLARE(STORAGE_DRIVER_S3_TYPE_STR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Defaults
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_DRIVER_S3_PORT_DEFAULT 443
|
||||
#define STORAGE_DRIVER_S3_TIMEOUT_DEFAULT 60000
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructor
|
||||
***********************************************************************************************************************************/
|
||||
StorageDriverS3 * storageDriverS3New(
|
||||
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
|
||||
const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey,
|
||||
const String *securityToken, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile,
|
||||
const String *caPath);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
bool storageDriverS3Exists(StorageDriverS3 *this, const String *path);
|
||||
StorageInfo storageDriverS3Info(StorageDriverS3 *this, const String *file, bool ignoreMissing);
|
||||
StringList *storageDriverS3List(StorageDriverS3 *this, const String *path, bool errorOnMissing, const String *expression);
|
||||
StorageFileRead *storageDriverS3NewRead(StorageDriverS3 *this, const String *file, bool ignoreMissing);
|
||||
StorageFileWrite *storageDriverS3NewWrite(
|
||||
StorageDriverS3 *this, const String *file, mode_t modeFile, mode_t modePath, bool createPath, bool syncFile, bool syncPath,
|
||||
bool atomic);
|
||||
void storageDriverS3PathCreate(StorageDriverS3 *this, const String *path, bool errorOnExists, bool noParentCreate, mode_t mode);
|
||||
void storageDriverS3PathRemove(StorageDriverS3 *this, const String *path, bool errorOnMissing, bool recurse);
|
||||
void storageDriverS3PathSync(StorageDriverS3 *this, const String *path, bool ignoreMissing);
|
||||
void storageDriverS3Remove(StorageDriverS3 *this, const String *file, bool errorOnMissing);
|
||||
|
||||
Buffer *storageDriverS3Request(
|
||||
StorageDriverS3 *this, const String *verb, const String *uri, const HttpQuery *query, const Buffer *body, bool returnContent,
|
||||
bool allowMissing);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Getters
|
||||
***********************************************************************************************************************************/
|
||||
HttpClient *storageDriverS3HttpClient(const StorageDriverS3 *this);
|
||||
Storage *storageDriverS3Interface(const StorageDriverS3 *this);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Destructor
|
||||
***********************************************************************************************************************************/
|
||||
void storageDriverS3Free(StorageDriverS3 *this);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Macros for function logging
|
||||
***********************************************************************************************************************************/
|
||||
#define FUNCTION_DEBUG_STORAGE_DRIVER_S3_TYPE \
|
||||
StorageDriverS3 *
|
||||
#define FUNCTION_DEBUG_STORAGE_DRIVER_S3_FORMAT(value, buffer, bufferSize) \
|
||||
objToLog(value, "StorageDriverS3", buffer, bufferSize)
|
||||
|
||||
#endif
|
||||
@ -8,6 +8,7 @@ Storage Helper
|
||||
#include "common/regExp.h"
|
||||
#include "config/config.h"
|
||||
#include "storage/driver/posix/storage.h"
|
||||
#include "storage/driver/s3/storage.h"
|
||||
#include "storage/helper.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -189,6 +190,18 @@ storageRepoGet(const String *type, bool write)
|
||||
cfgOptionStr(cfgOptRepoPath), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, write,
|
||||
storageRepoPathExpression));
|
||||
}
|
||||
else if (strEqZ(type, STORAGE_TYPE_S3))
|
||||
{
|
||||
result = storageDriverS3Interface(
|
||||
storageDriverS3New(
|
||||
cfgOptionStr(cfgOptRepoPath), write, storageRepoPathExpression, cfgOptionStr(cfgOptRepoS3Bucket),
|
||||
cfgOptionStr(cfgOptRepoS3Endpoint), cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key),
|
||||
cfgOptionStr(cfgOptRepoS3KeySecret), cfgOptionTest(cfgOptRepoS3Token) ? cfgOptionStr(cfgOptRepoS3Token) : NULL,
|
||||
cfgOptionTest(cfgOptRepoS3Host) ? cfgOptionStr(cfgOptRepoS3Host) : NULL,
|
||||
STORAGE_DRIVER_S3_PORT_DEFAULT, STORAGE_DRIVER_S3_TIMEOUT_DEFAULT, cfgOptionBool(cfgOptRepoS3VerifySsl),
|
||||
cfgOptionTest(cfgOptRepoS3CaFile) ? cfgOptionStr(cfgOptRepoS3CaFile) : NULL,
|
||||
cfgOptionTest(cfgOptRepoS3CaPath) ? cfgOptionStr(cfgOptRepoS3CaPath) : NULL));
|
||||
}
|
||||
else
|
||||
THROW_FMT(AssertError, "invalid storage type '%s'", strPtr(type));
|
||||
|
||||
|
||||
@ -48,12 +48,11 @@ storageNew(
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_INTERFACE, interface);
|
||||
|
||||
FUNCTION_TEST_ASSERT(type != NULL);
|
||||
FUNCTION_TEST_ASSERT(path != NULL);
|
||||
FUNCTION_TEST_ASSERT(path != NULL && strSize(path) >= 1 && strPtr(path)[0] == '/');
|
||||
FUNCTION_TEST_ASSERT(driver != NULL);
|
||||
FUNCTION_TEST_ASSERT(interface.exists != NULL);
|
||||
FUNCTION_TEST_ASSERT(interface.info != NULL);
|
||||
FUNCTION_TEST_ASSERT(interface.list != NULL);
|
||||
FUNCTION_TEST_ASSERT(interface.move != NULL);
|
||||
FUNCTION_TEST_ASSERT(interface.newRead != NULL);
|
||||
FUNCTION_TEST_ASSERT(interface.newWrite != NULL);
|
||||
FUNCTION_TEST_ASSERT(interface.pathCreate != NULL);
|
||||
@ -294,6 +293,7 @@ storageMove(const Storage *this, StorageFileRead *source, StorageFileWrite *dest
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_FILE_READ, source);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_FILE_WRITE, destination);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this->interface.move != NULL);
|
||||
FUNCTION_TEST_ASSERT(source != NULL);
|
||||
FUNCTION_TEST_ASSERT(destination != NULL);
|
||||
FUNCTION_DEBUG_ASSERT(!storageFileReadIgnoreMissing(source));
|
||||
|
||||
@ -527,6 +527,16 @@ unit:
|
||||
storage/helper: full
|
||||
storage/storage: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: s3
|
||||
total: 3
|
||||
|
||||
coverage:
|
||||
storage/driver/s3/fileRead: full
|
||||
storage/driver/s3/storage: full
|
||||
storage/helper: full
|
||||
storage/storage: full
|
||||
|
||||
# ********************************************************************************************************************************
|
||||
- name: protocol
|
||||
|
||||
|
||||
419
test/src/module/storage/s3Test.c
Normal file
419
test/src/module/storage/s3Test.c
Normal file
@ -0,0 +1,419 @@
|
||||
/***********************************************************************************************************************************
|
||||
Test S3 Storage Driver
|
||||
***********************************************************************************************************************************/
|
||||
#include <unistd.h>
|
||||
|
||||
#include "common/harnessConfig.h"
|
||||
#include "common/harnessTls.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Test server
|
||||
***********************************************************************************************************************************/
|
||||
#define DATE_REPLACE "????????"
|
||||
#define DATETIME_REPLACE "????????T??????Z"
|
||||
#define SHA256_REPLACE \
|
||||
"????????????????????????????????????????????????????????????????"
|
||||
|
||||
static const char *
|
||||
testS3ServerRequest(const char *verb, const char *uri)
|
||||
{
|
||||
String *request = strNewFmt(
|
||||
"%s %s HTTP/1.1\r\n"
|
||||
"authorization:AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/" DATE_REPLACE "/us-east-1/s3/aws4_request,"
|
||||
"SignedHeaders=content-length;host;x-amz-content-sha256;x-amz-date,Signature=" SHA256_REPLACE "\r\n"
|
||||
"content-length:%u\r\n"
|
||||
"host:" TLS_TEST_HOST "\r\n"
|
||||
"x-amz-content-sha256:%s\r\n"
|
||||
"x-amz-date:" DATETIME_REPLACE "\r\n"
|
||||
"\r\n",
|
||||
verb, uri, (unsigned int)0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||||
|
||||
return strPtr(request);
|
||||
}
|
||||
|
||||
static const char *
|
||||
testS3ServerResponse(unsigned int code, const char *message, const char *content)
|
||||
{
|
||||
String *response = strNewFmt("HTTP/1.1 %u %s\r\n", code, message);
|
||||
|
||||
if (content != NULL)
|
||||
{
|
||||
strCatFmt(
|
||||
response,
|
||||
"content-length:%zu\r\n"
|
||||
"\r\n"
|
||||
"%s",
|
||||
strlen(content), content);
|
||||
}
|
||||
else
|
||||
strCat(response, "\r\n");
|
||||
|
||||
return strPtr(response);
|
||||
}
|
||||
|
||||
static void
|
||||
testS3Server(void)
|
||||
{
|
||||
if (fork() == 0)
|
||||
{
|
||||
harnessTlsServerInit(TLS_TEST_PORT, TLS_CERT_TEST_CERT, TLS_CERT_TEST_KEY);
|
||||
harnessTlsServerAccept();
|
||||
|
||||
// storageDriverS3NewRead() and StorageDriverS3FileRead
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
// Ignore missing file
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt"));
|
||||
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL));
|
||||
|
||||
// Error on missing file
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt"));
|
||||
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL));
|
||||
|
||||
// Get file
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt"));
|
||||
harnessTlsServerReply(testS3ServerResponse(200, "OK", "this is a sample file"));
|
||||
|
||||
// Throw non-404 error
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt"));
|
||||
harnessTlsServerReply(testS3ServerResponse(303, "Some bad status", "CONTENT"));
|
||||
|
||||
// storageDriverList()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
// Throw error
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"));
|
||||
harnessTlsServerReply(testS3ServerResponse(344, "Another bad status", NULL));
|
||||
|
||||
// list a file/path in root
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"));
|
||||
harnessTlsServerReply(
|
||||
testS3ServerResponse(
|
||||
200, "OK",
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
|
||||
" <Contents>"
|
||||
" <Key>test1.txt</Key>"
|
||||
" </Contents>"
|
||||
" <CommonPrefixes>"
|
||||
" <Prefix>path1/</Prefix>"
|
||||
" </CommonPrefixes>"
|
||||
"</ListBucketResult>"));
|
||||
|
||||
// list a file in root with expression
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test"));
|
||||
harnessTlsServerReply(
|
||||
testS3ServerResponse(
|
||||
200, "OK",
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
|
||||
" <Contents>"
|
||||
" <Key>test1.txt</Key>"
|
||||
" </Contents>"
|
||||
"</ListBucketResult>"));
|
||||
|
||||
// list files with continuation
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F"));
|
||||
harnessTlsServerReply(
|
||||
testS3ServerResponse(
|
||||
200, "OK",
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
|
||||
" <NextContinuationToken>1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=</NextContinuationToken>"
|
||||
" <Contents>"
|
||||
" <Key>path/to/test1.txt</Key>"
|
||||
" </Contents>"
|
||||
" <Contents>"
|
||||
" <Key>path/to/test2.txt</Key>"
|
||||
" </Contents>"
|
||||
" <CommonPrefixes>"
|
||||
" <Prefix>path/to/path1/</Prefix>"
|
||||
" </CommonPrefixes>"
|
||||
"</ListBucketResult>"));
|
||||
|
||||
harnessTlsServerExpect(
|
||||
testS3ServerRequest(
|
||||
HTTP_VERB_GET,
|
||||
"/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2"
|
||||
"&prefix=path%2Fto%2F"));
|
||||
harnessTlsServerReply(
|
||||
testS3ServerResponse(
|
||||
200, "OK",
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
|
||||
" <Contents>"
|
||||
" <Key>path/to/test3.txt</Key>"
|
||||
" </Contents>"
|
||||
" <CommonPrefixes>"
|
||||
" <Prefix>path/to/path2/</Prefix>"
|
||||
" </CommonPrefixes>"
|
||||
"</ListBucketResult>"));
|
||||
|
||||
// list files with expression
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest"));
|
||||
harnessTlsServerReply(
|
||||
testS3ServerResponse(
|
||||
200, "OK",
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
|
||||
" <Contents>"
|
||||
" <Key>path/to/test1.txt</Key>"
|
||||
" </Contents>"
|
||||
" <Contents>"
|
||||
" <Key>path/to/test2.txt</Key>"
|
||||
" </Contents>"
|
||||
" <Contents>"
|
||||
" <Key>path/to/test3.txt</Key>"
|
||||
" </Contents>"
|
||||
" <CommonPrefixes>"
|
||||
" <Prefix>path/to/test1.path/</Prefix>"
|
||||
" </CommonPrefixes>"
|
||||
" <CommonPrefixes>"
|
||||
" <Prefix>path/to/test2.path/</Prefix>"
|
||||
" </CommonPrefixes>"
|
||||
"</ListBucketResult>"));
|
||||
|
||||
harnessTlsServerClose();
|
||||
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Test Run
|
||||
***********************************************************************************************************************************/
|
||||
void
|
||||
testRun(void)
|
||||
{
|
||||
FUNCTION_HARNESS_VOID();
|
||||
|
||||
// Test strings
|
||||
const String *path = strNew("/");
|
||||
const String *bucket = strNew("bucket");
|
||||
const String *region = strNew("us-east-1");
|
||||
const String *endPoint = strNew("s3.amazonaws.com");
|
||||
const String *host = strNew(TLS_TEST_HOST);
|
||||
const unsigned int port = TLS_TEST_PORT;
|
||||
const String *accessKey = strNew("AKIAIOSFODNN7EXAMPLE");
|
||||
const String *secretAccessKey = strNew("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
|
||||
const String *securityToken = strNew(
|
||||
"AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/q"
|
||||
"kPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xV"
|
||||
"qr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==");
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("storageDriverS3New() and storageRepoGet()"))
|
||||
{
|
||||
// Only required options
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
StringList *argList = strLstNew();
|
||||
strLstAddZ(argList, "pgbackrest");
|
||||
strLstAddZ(argList, "--stanza=db");
|
||||
strLstAddZ(argList, "--repo1-type=s3");
|
||||
strLstAdd(argList, strNewFmt("--repo1-path=%s", strPtr(path)));
|
||||
strLstAdd(argList, strNewFmt("--repo1-s3-bucket=%s", strPtr(bucket)));
|
||||
strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region)));
|
||||
strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s", strPtr(endPoint)));
|
||||
setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true);
|
||||
setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true);
|
||||
strLstAddZ(argList, "archive-get");
|
||||
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
|
||||
|
||||
Storage *storage = NULL;
|
||||
TEST_ASSIGN(storage, storageRepoGet(strNew(STORAGE_TYPE_S3), false), "get S3 repo storage");
|
||||
TEST_RESULT_STR(strPtr(storage->path), strPtr(path), " check path");
|
||||
TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->bucket), strPtr(bucket), " check bucket");
|
||||
TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->region), strPtr(region), " check region");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(((StorageDriverS3 *)storage->driver)->host), strPtr(strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint))),
|
||||
" check host");
|
||||
TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->accessKey), strPtr(accessKey), " check access key");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(((StorageDriverS3 *)storage->driver)->secretAccessKey), strPtr(secretAccessKey), " check secret access key");
|
||||
TEST_RESULT_PTR(((StorageDriverS3 *)storage->driver)->securityToken, NULL, " check security token");
|
||||
|
||||
// Add default options
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
argList = strLstNew();
|
||||
strLstAddZ(argList, "pgbackrest");
|
||||
strLstAddZ(argList, "--stanza=db");
|
||||
strLstAddZ(argList, "--repo1-type=s3");
|
||||
strLstAdd(argList, strNewFmt("--repo1-path=%s", strPtr(path)));
|
||||
strLstAdd(argList, strNewFmt("--repo1-s3-bucket=%s", strPtr(bucket)));
|
||||
strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region)));
|
||||
strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s", strPtr(endPoint)));
|
||||
strLstAdd(argList, strNewFmt("--repo1-s3-host=%s", strPtr(host)));
|
||||
strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH);
|
||||
strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_FAKE_PATH "/pgbackrest-test.crt");
|
||||
setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true);
|
||||
setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true);
|
||||
setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true);
|
||||
strLstAddZ(argList, "archive-get");
|
||||
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
|
||||
|
||||
TEST_ASSIGN(storage, storageRepoGet(strNew(STORAGE_TYPE_S3), false), "get S3 repo storage with options");
|
||||
TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->bucket), strPtr(bucket), " check bucket");
|
||||
TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->region), strPtr(region), " check region");
|
||||
TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->host), strPtr(host), " check host");
|
||||
TEST_RESULT_STR(strPtr(((StorageDriverS3 *)storage->driver)->accessKey), strPtr(accessKey), " check access key");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(((StorageDriverS3 *)storage->driver)->secretAccessKey), strPtr(secretAccessKey), " check secret access key");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(((StorageDriverS3 *)storage->driver)->securityToken), strPtr(securityToken), " check security token");
|
||||
}
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("storageDriverS3DateTime() and storageDriverS3Auth()"))
|
||||
{
|
||||
TEST_RESULT_STR(strPtr(storageDriverS3DateTime(1491267845)), "20170404T010405Z", "static date");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
StorageDriverS3 *driver = storageDriverS3New(
|
||||
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, NULL, 0, 0, true, NULL, NULL);
|
||||
|
||||
HttpHeader *header = httpHeaderNew(NULL);
|
||||
|
||||
HttpQuery *query = httpQueryNew();
|
||||
httpQueryAdd(query, strNew("list-type"), strNew("2"));
|
||||
|
||||
TEST_RESULT_VOID(
|
||||
storageDriverS3Auth(
|
||||
driver, strNew("GET"), strNew("/"), query, strNew("20170606T121212Z"), header,
|
||||
strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")),
|
||||
"generate authorization");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(httpHeaderGet(header, strNew("authorization"))),
|
||||
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,"
|
||||
"SignedHeaders=host;x-amz-content-sha256;x-amz-date,"
|
||||
"Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3",
|
||||
" check authorization header");
|
||||
|
||||
// Test again to be sure cache signing key is used
|
||||
const Buffer *lastSigningKey = driver->signingKey;
|
||||
|
||||
TEST_RESULT_VOID(
|
||||
storageDriverS3Auth(
|
||||
driver, strNew("GET"), strNew("/"), query, strNew("20170606T121212Z"), header,
|
||||
strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")),
|
||||
"generate authorization");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(httpHeaderGet(header, strNew("authorization"))),
|
||||
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,"
|
||||
"SignedHeaders=host;x-amz-content-sha256;x-amz-date,"
|
||||
"Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3",
|
||||
" check authorization header");
|
||||
TEST_RESULT_BOOL(driver->signingKey == lastSigningKey, true, " check signing key was reused");
|
||||
|
||||
// Change the date to generate a new signing key
|
||||
TEST_RESULT_VOID(
|
||||
storageDriverS3Auth(
|
||||
driver, strNew("GET"), strNew("/"), query, strNew("20180814T080808Z"), header,
|
||||
strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")),
|
||||
" generate authorization");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(httpHeaderGet(header, strNew("authorization"))),
|
||||
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20180814/us-east-1/s3/aws4_request,"
|
||||
"SignedHeaders=host;x-amz-content-sha256;x-amz-date,"
|
||||
"Signature=d0fa9c36426eb94cdbaf287a7872c7a3b6c913f523163d0d7debba0758e36f49",
|
||||
" check authorization header");
|
||||
TEST_RESULT_BOOL(driver->signingKey != lastSigningKey, true, " check signing key was regenerated");
|
||||
|
||||
// Test with security token
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
driver = storageDriverS3New(
|
||||
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, securityToken, NULL, 0, 0, true, NULL, NULL);
|
||||
|
||||
TEST_RESULT_VOID(
|
||||
storageDriverS3Auth(
|
||||
driver, strNew("GET"), strNew("/"), query, strNew("20170606T121212Z"), header,
|
||||
strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")),
|
||||
"generate authorization");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(httpHeaderGet(header, strNew("authorization"))),
|
||||
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,"
|
||||
"SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,"
|
||||
"Signature=c12565bf5d7e0ef623f76d66e09e5431aebef803f6a25a01c586525f17e474a3",
|
||||
" check authorization header");
|
||||
}
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("storageDriverS3*() and StorageDriverS3FileRead"))
|
||||
{
|
||||
testS3Server();
|
||||
|
||||
StorageDriverS3 *s3Driver = storageDriverS3New(
|
||||
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, host, port, 250, true, NULL, NULL);
|
||||
Storage *s3 = storageDriverS3Interface(s3Driver);
|
||||
|
||||
// Coverage for noop functions
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_RESULT_VOID(storagePathCreateNP(s3, strNew("path")), "path create is a noop");
|
||||
TEST_RESULT_VOID(storagePathSyncNP(s3, strNew("path")), "path sync is a noop");
|
||||
|
||||
// storageDriverS3NewRead() and StorageDriverS3FileRead
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_RESULT_PTR(
|
||||
storageGetNP(storageNewReadP(s3, strNew("file.txt"), .ignoreMissing = true)), NULL, "ignore missing file");
|
||||
TEST_ERROR(
|
||||
storageGetNP(storageNewReadNP(s3, strNew("file.txt"))), FileMissingError,
|
||||
"unable to open '/file.txt': No such file or directory");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(strNewBuf(storageGetNP(storageNewReadNP(s3, strNew("file.txt"))))), "this is a sample file",
|
||||
"get file");
|
||||
|
||||
StorageFileRead *read = NULL;
|
||||
TEST_ASSIGN(read, storageNewReadP(s3, strNew("file.txt"), .ignoreMissing = true), "new read file");
|
||||
TEST_RESULT_BOOL(storageFileReadIgnoreMissing(read), true, " check ignore missing");
|
||||
TEST_RESULT_STR(strPtr(storageFileReadName(read)), "/file.txt", " check name");
|
||||
|
||||
TEST_ERROR(
|
||||
ioReadOpen(storageFileReadIo(read)), ProtocolError,
|
||||
"S3 request failed with 303: Some bad status\n"
|
||||
"*** URI/Query ***:\n"
|
||||
"/file.txt\n"
|
||||
"*** Request Headers ***:\n"
|
||||
"authorization: <redacted>\n"
|
||||
"content-length: 0\n"
|
||||
"host: " TLS_TEST_HOST "\n"
|
||||
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
|
||||
"x-amz-date: <redacted>\n"
|
||||
"*** Response Headers ***:\n"
|
||||
"content-length: 7\n"
|
||||
"*** Response Content ***:\n"
|
||||
"CONTENT")
|
||||
|
||||
// storageDriverList()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_ERROR(
|
||||
storageListP(s3, strNew("/"), .errorOnMissing = true), AssertError, "function test assertion '!errorOnMissing' failed");
|
||||
TEST_ERROR(storageListNP(s3, strNew("/")), ProtocolError,
|
||||
"S3 request failed with 344: Another bad status\n"
|
||||
"*** URI/Query ***:\n"
|
||||
"/?delimiter=%2F&list-type=2\n"
|
||||
"*** Request Headers ***:\n"
|
||||
"authorization: <redacted>\n"
|
||||
"content-length: 0\n"
|
||||
"host: " TLS_TEST_HOST "\n"
|
||||
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
|
||||
"x-amz-date: <redacted>");
|
||||
|
||||
TEST_RESULT_STR(strPtr(strLstJoin(storageListNP(s3, strNew("/")), ",")), "path1,test1.txt", "list a file/path in root");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(strLstJoin(storageListP(s3, strNew("/"), .expression = strNew("^test.*$")), ",")), "test1.txt",
|
||||
"list a file in root with expression");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(strLstJoin(storageListNP(s3, strNew("/path/to")), ",")),
|
||||
"path1,test1.txt,test2.txt,path2,test3.txt", "list files with continuation");
|
||||
TEST_RESULT_STR(
|
||||
strPtr(strLstJoin(storageListP(s3, strNew("/path/to"), .expression = strNew("^test(1|3)")), ",")),
|
||||
"test1.path,test1.txt,test3.txt", "list files with expression");
|
||||
|
||||
// Coverage for unimplemented functions
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_ERROR(storageExistsNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
|
||||
TEST_ERROR(storageInfoNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
|
||||
TEST_ERROR(storageNewWriteNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
|
||||
TEST_ERROR(storagePathRemoveNP(s3, strNew("path")), AssertError, "NOT YET IMPLEMENTED");
|
||||
TEST_ERROR(storageRemoveNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
|
||||
}
|
||||
|
||||
FUNCTION_HARNESS_RESULT_VOID();
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user