db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-05 15:05:48 +02:00
Code Issues Releases Activity

Add file write to the S3 storage driver.

Browse Source 
Now that repositories are writable the storage drivers that don't yet support file writes need to be updated to do so.

Note that the part size for multi-part upload has not been defined as a proper constant.  This will become an option in the near future so it doesn't seem worth creating a constant that we might then forget to remove.
This commit is contained in:
David Steele 2019-03-17 22:00:54 +04:00
parent 7193738288
commit 856a369b86
9 changed files with 705 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/xml/release.xml
View File

@ -42,7 +42,7 @@
</release-item>
<release-item>
<p>Add file write to the remote storage driver.</p>
<p>Add file write to the remote and S3 storage drivers.</p>
</release-item>
<release-item>

6
src/Makefile
View File

@ -159,6 +159,7 @@ SRCS = \
storage/driver/remote/protocol.c \
storage/driver/remote/storage.c \
storage/driver/s3/fileRead.c \
storage/driver/s3/fileWrite.c \
storage/driver/s3/storage.c \
storage/fileRead.c \
storage/fileWrite.c \
@ -503,7 +504,10 @@ storage/driver/remote/storage.o: storage/driver/remote/storage.c common/assert.h
storage/driver/s3/fileRead.o: storage/driver/s3/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h storage/driver/s3/fileRead.h storage/driver/s3/storage.h storage/fileRead.h storage/fileRead.intern.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
$(CC) $(CFLAGS) -c storage/driver/s3/fileRead.c -o storage/driver/s3/fileRead.o
storage/driver/s3/storage.o: storage/driver/s3/storage.c common/assert.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/common.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/type/xml.h storage/driver/s3/fileRead.h storage/driver/s3/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
storage/driver/s3/fileWrite.o: storage/driver/s3/fileWrite.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/type/xml.h storage/driver/s3/fileWrite.h storage/driver/s3/storage.h storage/fileRead.h storage/fileWrite.h storage/fileWrite.intern.h storage/info.h storage/storage.h storage/storage.intern.h version.h
$(CC) $(CFLAGS) -c storage/driver/s3/fileWrite.c -o storage/driver/s3/fileWrite.o
storage/driver/s3/storage.o: storage/driver/s3/storage.c common/assert.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/common.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/type/xml.h storage/driver/s3/fileRead.h storage/driver/s3/fileWrite.h storage/driver/s3/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
$(CC) $(CFLAGS) -c storage/driver/s3/storage.c -o storage/driver/s3/storage.o
storage/fileRead.o: storage/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/fileRead.h storage/fileRead.intern.h

414
src/storage/driver/s3/fileWrite.c Normal file
View File

@ -0,0 +1,414 @@
/***********************************************************************************************************************************
S3 Storage File Write Driver
***********************************************************************************************************************************/
#include "common/debug.h"
#include "common/io/write.intern.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/xml.h"
#include "storage/driver/s3/fileWrite.h"
#include "storage/fileWrite.intern.h"
/***********************************************************************************************************************************
S3 query tokens
***********************************************************************************************************************************/
STRING_STATIC(S3_QUERY_PART_NUMBER_STR, "partNumber");
STRING_STATIC(S3_QUERY_UPLOADS_STR, "uploads");
STRING_STATIC(S3_QUERY_UPLOAD_ID_STR, "uploadId");
/***********************************************************************************************************************************
XML tags
***********************************************************************************************************************************/
STRING_STATIC(S3_XML_TAG_ETAG_STR, "ETag");
STRING_STATIC(S3_XML_TAG_UPLOAD_ID_STR, "UploadId");
STRING_STATIC(S3_XML_TAG_COMPLETE_MULTIPART_UPLOAD_STR, "CompleteMultipartUpload");
STRING_STATIC(S3_XML_TAG_PART_STR, "Part");
STRING_STATIC(S3_XML_TAG_PART_NUMBER_STR, "PartNumber");
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct StorageDriverS3FileWrite
{
MemContext *memContext;
StorageDriverS3 *storage;
StorageFileWrite *interface;
IoWrite *io;
const String *path;
const String *name;
size_t partSize;
Buffer *partBuffer;
const String *uploadId;
StringList *uploadPartList;
};
/***********************************************************************************************************************************
Create a new file
***********************************************************************************************************************************/
StorageDriverS3FileWrite *
storageDriverS3FileWriteNew(StorageDriverS3 *storage, const String *name, size_t partSize)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_DRIVER_S3, storage);
FUNCTION_LOG_PARAM(STRING, name);
FUNCTION_LOG_END();
ASSERT(storage != NULL);
ASSERT(name != NULL);
StorageDriverS3FileWrite *this = NULL;
// Create the file
MEM_CONTEXT_NEW_BEGIN("StorageDriverS3FileWrite")
{
this = memNew(sizeof(StorageDriverS3FileWrite));
this->memContext = MEM_CONTEXT_NEW();
this->storage = storage;
this->interface = storageFileWriteNewP(
STORAGE_DRIVER_S3_TYPE_STR, this, .atomic = (StorageFileWriteInterfaceAtomic)storageDriverS3FileWriteAtomic,
.createPath = (StorageFileWriteInterfaceCreatePath)storageDriverS3FileWriteCreatePath,
.io = (StorageFileWriteInterfaceIo)storageDriverS3FileWriteIo,
.modeFile = (StorageFileWriteInterfaceModeFile)storageDriverS3FileWriteModeFile,
.modePath = (StorageFileWriteInterfaceModePath)storageDriverS3FileWriteModePath,
.name = (StorageFileWriteInterfaceName)storageDriverS3FileWriteName,
.syncFile = (StorageFileWriteInterfaceSyncFile)storageDriverS3FileWriteSyncFile,
.syncPath = (StorageFileWriteInterfaceSyncPath)storageDriverS3FileWriteSyncPath);
this->io = ioWriteNewP(
this, .close = (IoWriteInterfaceClose)storageDriverS3FileWriteClose,
.open = (IoWriteInterfaceOpen)storageDriverS3FileWriteOpen,
.write = (IoWriteInterfaceWrite)storageDriverS3FileWrite);
this->path = strPath(name);
this->name = strDup(name);
this->partSize = partSize;
}
MEM_CONTEXT_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_DRIVER_S3_FILE_WRITE, this);
}
/***********************************************************************************************************************************
Open the file
***********************************************************************************************************************************/
void
storageDriverS3FileWriteOpen(StorageDriverS3FileWrite *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(this->partBuffer == NULL);
// Allocate the part buffer
MEM_CONTEXT_BEGIN(this->memContext)
{
this->partBuffer = bufNew(this->partSize);
}
MEM_CONTEXT_END();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Flush bytes to upload part
***********************************************************************************************************************************/
static void
storageDriverS3FileWritePart(StorageDriverS3FileWrite *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(this->partBuffer != NULL);
ASSERT(bufSize(this->partBuffer) > 0);
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the upload id if we have not already
if (this->uploadId == NULL)
{
// Initiate mult-part upload
XmlNode *xmlRoot = xmlDocumentRoot(
xmlDocumentNewBuf(
storageDriverS3Request(
this->storage, HTTP_VERB_POST_STR, this->name,
httpQueryAdd(httpQueryNew(), S3_QUERY_UPLOADS_STR, EMPTY_STR), NULL, true, false).response));
// Store the upload id
MEM_CONTEXT_BEGIN(this->memContext)
{
this->uploadId = xmlNodeContent(xmlNodeChild(xmlRoot, S3_XML_TAG_UPLOAD_ID_STR, true));
this->uploadPartList = strLstNew();
}
MEM_CONTEXT_END();
}
// Upload the part and add etag to part list
HttpQuery *query = httpQueryNew();
httpQueryAdd(query, S3_QUERY_UPLOAD_ID_STR, this->uploadId);
httpQueryAdd(query, S3_QUERY_PART_NUMBER_STR, strNewFmt("%u", strLstSize(this->uploadPartList) + 1));
strLstAdd(
this->uploadPartList,
httpHeaderGet(
storageDriverS3Request(
this->storage, HTTP_VERB_PUT_STR, this->name, query, this->partBuffer, true, false).responseHeader,
HTTP_HEADER_ETAG_STR));
ASSERT(strLstGet(this->uploadPartList, strLstSize(this->uploadPartList) - 1) != NULL);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Write to internal buffer
***********************************************************************************************************************************/
void
storageDriverS3FileWrite(StorageDriverS3FileWrite *this, const Buffer *buffer)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_LOG_PARAM(BUFFER, buffer);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(this->partBuffer != NULL);
size_t bytesTotal = 0;
// Continue until the write buffer has been exhausted
do
{
// Copy an many bytes as possible into the part buffer
size_t bytesNext = bufRemains(this->partBuffer) > bufUsed(buffer) - bytesTotal ?
bufUsed(buffer) - bytesTotal : bufRemains(this->partBuffer);
bufCatSub(this->partBuffer, buffer, bytesTotal, bytesNext);
bytesTotal += bytesNext;
// If the part buffer is full then write it
if (bufRemains(this->partBuffer) == 0)
{
storageDriverS3FileWritePart(this);
bufUsedZero(this->partBuffer);
}
}
while (bytesTotal != bufUsed(buffer));
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Close the file
***********************************************************************************************************************************/
void
storageDriverS3FileWriteClose(StorageDriverS3FileWrite *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
// Close if the file has not already been closed
if (this->partBuffer != NULL)
{
MEM_CONTEXT_TEMP_BEGIN()
{
// If a multi-part upload was started we need to finish that way
if (this->uploadId != NULL)
{
// If there is anything left in the part buffer then write it
if (bufUsed(this->partBuffer) > 0)
storageDriverS3FileWritePart(this);
// Generate the xml part list
XmlDocument *partList = xmlDocumentNew(S3_XML_TAG_COMPLETE_MULTIPART_UPLOAD_STR);
for (unsigned int partIdx = 0; partIdx < strLstSize(this->uploadPartList); partIdx++)
{
XmlNode *partNode = xmlNodeAdd(xmlDocumentRoot(partList), S3_XML_TAG_PART_STR);
xmlNodeContentSet(xmlNodeAdd(partNode, S3_XML_TAG_PART_NUMBER_STR), strNewFmt("%u", partIdx + 1));
xmlNodeContentSet(xmlNodeAdd(partNode, S3_XML_TAG_ETAG_STR), strLstGet(this->uploadPartList, partIdx));
}
// Finalize the multi-part upload
storageDriverS3Request(
this->storage, HTTP_VERB_POST_STR, this->name,
httpQueryAdd(httpQueryNew(), S3_QUERY_UPLOAD_ID_STR, this->uploadId), xmlDocumentBuf(partList), false, false);
}
// Else upload all the data in a single put
else
storageDriverS3Request(this->storage, HTTP_VERB_PUT_STR, this->name, NULL, this->partBuffer, false, false);
bufFree(this->partBuffer);
this->partBuffer = NULL;
}
MEM_CONTEXT_TEMP_END();
}
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
S3 operations are always atomic, so return true
***********************************************************************************************************************************/
bool
storageDriverS3FileWriteAtomic(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
(void)this;
FUNCTION_TEST_RETURN(true);
}
/***********************************************************************************************************************************
S3 paths are always implicitly created
***********************************************************************************************************************************/
bool
storageDriverS3FileWriteCreatePath(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
(void)this;
FUNCTION_TEST_RETURN(true);
}
/***********************************************************************************************************************************
Get interface
***********************************************************************************************************************************/
StorageFileWrite *
storageDriverS3FileWriteInterface(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->interface);
}
/***********************************************************************************************************************************
Get I/O interface
***********************************************************************************************************************************/
IoWrite *
storageDriverS3FileWriteIo(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->io);
}
/***********************************************************************************************************************************
S3 does not support Posix-style mode
***********************************************************************************************************************************/
mode_t
storageDriverS3FileWriteModeFile(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
(void)this;
FUNCTION_TEST_RETURN(0);
}
/***********************************************************************************************************************************
S3 does not support Posix-style mode
***********************************************************************************************************************************/
mode_t
storageDriverS3FileWriteModePath(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
(void)this;
FUNCTION_TEST_RETURN(0);
}
/***********************************************************************************************************************************
File name
***********************************************************************************************************************************/
const String *
storageDriverS3FileWriteName(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->name);
}
/***********************************************************************************************************************************
S3 operations are always atomic, so return true
***********************************************************************************************************************************/
bool
storageDriverS3FileWriteSyncFile(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
(void)this;
FUNCTION_TEST_RETURN(true);
}
/***********************************************************************************************************************************
S3 operations are always atomic, so return true
***********************************************************************************************************************************/
bool
storageDriverS3FileWriteSyncPath(const StorageDriverS3FileWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
(void)this;
FUNCTION_TEST_RETURN(true);
}
/***********************************************************************************************************************************
Free the file
***********************************************************************************************************************************/
void
storageDriverS3FileWriteFree(StorageDriverS3FileWrite *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_DRIVER_S3_FILE_WRITE, this);
FUNCTION_LOG_END();
if (this != NULL)
memContextFree(this->memContext);
FUNCTION_LOG_RETURN_VOID();
}

57
src/storage/driver/s3/fileWrite.h Normal file
View File

@ -0,0 +1,57 @@
/***********************************************************************************************************************************
S3 Storage File Write Driver
***********************************************************************************************************************************/
#ifndef STORAGE_DRIVER_S3_FILEWRITE_H
#define STORAGE_DRIVER_S3_FILEWRITE_H
#include <sys/types.h>
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct StorageDriverS3FileWrite StorageDriverS3FileWrite;
#include "common/type/buffer.h"
#include "storage/driver/s3/storage.h"
#include "storage/fileWrite.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
StorageDriverS3FileWrite *storageDriverS3FileWriteNew(StorageDriverS3 *storage, const String *name, size_t partSize);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void storageDriverS3FileWriteOpen(StorageDriverS3FileWrite *this);
void storageDriverS3FileWrite(StorageDriverS3FileWrite *this, const Buffer *buffer);
void storageDriverS3FileWriteClose(StorageDriverS3FileWrite *this);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool storageDriverS3FileWriteAtomic(const StorageDriverS3FileWrite *this);
bool storageDriverS3FileWriteCreatePath(const StorageDriverS3FileWrite *this);
mode_t storageDriverS3FileWriteModeFile(const StorageDriverS3FileWrite *this);
StorageFileWrite* storageDriverS3FileWriteInterface(const StorageDriverS3FileWrite *this);
IoWrite *storageDriverS3FileWriteIo(const StorageDriverS3FileWrite *this);
mode_t storageDriverS3FileWriteModePath(const StorageDriverS3FileWrite *this);
const String *storageDriverS3FileWriteName(const StorageDriverS3FileWrite *this);
const StorageDriverS3 *storageDriverS3FileWriteStorage(const StorageDriverS3FileWrite *this);
bool storageDriverS3FileWriteSyncFile(const StorageDriverS3FileWrite *this);
bool storageDriverS3FileWriteSyncPath(const StorageDriverS3FileWrite *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void storageDriverS3FileWriteFree(StorageDriverS3FileWrite *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_LOG_STORAGE_DRIVER_S3_FILE_WRITE_TYPE \
StorageDriverS3FileWrite *
#define FUNCTION_LOG_STORAGE_DRIVER_S3_FILE_WRITE_FORMAT(value, buffer, bufferSize) \
objToLog(value, "StorageDriverS3FileWrite", buffer, bufferSize)
#endif

60
src/storage/driver/s3/storage.c
View File

@ -4,6 +4,7 @@ S3 Storage Driver
#include <time.h>
#include "common/crypto/hash.h"
#include "common/encode.h"
#include "common/debug.h"
#include "common/io/http/common.h"
#include "common/log.h"
@ -11,6 +12,7 @@ S3 Storage Driver
#include "common/regExp.h"
#include "common/type/xml.h"
#include "storage/driver/s3/fileRead.h"
#include "storage/driver/s3/fileWrite.h"
#include "storage/driver/s3/storage.h"
/***********************************************************************************************************************************
@ -22,9 +24,9 @@ STRING_EXTERN(STORAGE_DRIVER_S3_TYPE_STR, STORAGE_DRIV
S3 http headers
***********************************************************************************************************************************/
STRING_STATIC(S3_HEADER_AUTHORIZATION_STR, "authorization");
STRING_STATIC(S3_HEADER_HOST_STR, "host");
STRING_STATIC(S3_HEADER_CONTENT_SHA256_STR, "x-amz-content-sha256");
STRING_STATIC(S3_HEADER_DATE_STR, "x-amz-date");
STRING_STATIC(S3_HEADER_HOST_STR, "host");
STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-security-token");
/***********************************************************************************************************************************
@ -74,6 +76,7 @@ struct StorageDriverS3
const String *accessKey; // Access key
const String *secretAccessKey; // Secret access key
const String *securityToken; // Security token, if any
size_t partSize; // Part size for multi-part upload
const String *host; // Defaults to {bucket}.{endpoint}
// Current signing key and date it is valid for
@ -177,7 +180,7 @@ storageDriverS3Auth(
// Generate signing key. This key only needs to be regenerated every seven days but we'll do it once a day to keep the
// logic simple. It's a relatively expensive operation so we'd rather not do it for every request.
// If the cached signing key has expired (or has noe been generated) then regenerate it
// If the cached signing key has expired (or has none been generated) then regenerate it
if (!strEq(date, this->signingKeyDate))
{
const Buffer *dateKey = cryptoHmacOne(
@ -214,8 +217,8 @@ StorageDriverS3 *
storageDriverS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey,
const String *securityToken, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile,
const String *caPath)
const String *securityToken, size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer,
const String *caFile, const String *caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@ -227,6 +230,7 @@ storageDriverS3New(
FUNCTION_TEST_PARAM(STRING, accessKey);
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
FUNCTION_TEST_PARAM(STRING, securityToken);
FUNCTION_TEST_PARAM(SIZE, partSize);
FUNCTION_LOG_PARAM(STRING, host);
FUNCTION_LOG_PARAM(UINT, port);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
@ -255,6 +259,7 @@ storageDriverS3New(
this->accessKey = strDup(accessKey);
this->secretAccessKey = strDup(secretAccessKey);
this->securityToken = strDup(securityToken);
this->partSize = partSize;
this->host = host == NULL ? strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint)) : strDup(host);
// Force the signing key to be generated on the first run
@ -282,7 +287,7 @@ storageDriverS3New(
/***********************************************************************************************************************************
Process S3 request
***********************************************************************************************************************************/
Buffer *
StorageDriverS3RequestResult
storageDriverS3Request(
StorageDriverS3 *this, const String *verb, const String *uri, const HttpQuery *query, const Buffer *body, bool returnContent,
bool allowMissing)
@ -301,21 +306,36 @@ storageDriverS3Request(
ASSERT(verb != NULL);
ASSERT(uri != NULL);
Buffer *result = NULL;
StorageDriverS3RequestResult result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
// Create header list and add content length
HttpHeader *requestHeader = httpHeaderNew(this->headerRedactList);
httpHeaderAdd(requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
// Set content length
httpHeaderAdd(
requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR,
body == NULL || bufUsed(body) == 0 ? ZERO_STR : strNewFmt("%zu", bufUsed(body)));
// Calculate content-md5 header if there is content
if (body != NULL)
{
char md5Hash[HASH_TYPE_MD5_SIZE_HEX];
encodeToStr(encodeBase64, bufPtr(cryptoHashOne(HASH_TYPE_MD5_STR, body)), HASH_TYPE_M5_SIZE, md5Hash);
httpHeaderAdd(requestHeader, HTTP_HEADER_CONTENT_MD5_STR, strNew(md5Hash));
}
// Generate authorization header
storageDriverS3Auth(
this, verb, uri, query, storageDriverS3DateTime(time(NULL)), requestHeader,
strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"));
body == NULL || bufUsed(body) == 0 ?
strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") :
bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR, body)));
// Process request
result = httpClientRequest(this->httpClient, verb, uri, query, requestHeader, body, returnContent);
Buffer *response = httpClientRequest(this->httpClient, verb, uri, query, requestHeader, body, returnContent);
// Error if the request was not successful
if (httpClientResponseCode(this->httpClient) != HTTP_RESPONSE_CODE_OK &&
@ -365,18 +385,19 @@ storageDriverS3Request(
}
// If there was content then output it
if (result != NULL)
strCatFmt(error, "\n*** Response Content ***:\n%s", strPtr(strNewBuf(result)));
if (response!= NULL)
strCatFmt(error, "\n*** Response Content ***:\n%s", strPtr(strNewBuf(response)));
THROW(ProtocolError, strPtr(error));
}
// On success move the buffer to the calling context
bufMove(result, MEM_CONTEXT_OLD());
result.responseHeader = httpHeaderMove(httpHeaderDup(httpClientReponseHeader(this->httpClient), NULL), MEM_CONTEXT_OLD());
result.response = bufMove(response, MEM_CONTEXT_OLD());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BUFFER, result);
FUNCTION_LOG_RETURN(STORAGE_DRIVER_S3_REQUEST_RESULT, result);
}
/***********************************************************************************************************************************
@ -407,7 +428,7 @@ storageDriverS3Exists(StorageDriverS3 *this, const String *path)
httpQueryAdd(query, S3_QUERY_LIST_TYPE_STR, S3_QUERY_VALUE_LIST_TYPE_2_STR);
XmlNode *xmlRoot = xmlDocumentRoot(
xmlDocumentNewBuf(storageDriverS3Request(this, HTTP_VERB_GET_STR, FSLASH_STR, query, NULL, true, false)));
xmlDocumentNewBuf(storageDriverS3Request(this, HTTP_VERB_GET_STR, FSLASH_STR, query, NULL, true, false).response));
// Check if the prefix exists. If not then the file definitely does not exist, but if it does we'll need to check the
// exact name to be sure we are not looking at a different file with the same prefix
@ -523,7 +544,8 @@ storageDriverS3List(StorageDriverS3 *this, const String *path, bool errorOnMissi
httpQueryAdd(query, S3_QUERY_PREFIX_STR, queryPrefix);
XmlNode *xmlRoot = xmlDocumentRoot(
xmlDocumentNewBuf(storageDriverS3Request(this, HTTP_VERB_GET_STR, FSLASH_STR, query, NULL, true, false)));
xmlDocumentNewBuf(
storageDriverS3Request(this, HTTP_VERB_GET_STR, FSLASH_STR, query, NULL, true, false).response));
// Get subpath list
XmlNodeList *subPathList = xmlNodeChildList(xmlRoot, S3_XML_TAG_COMMON_PREFIXES_STR);
@ -614,12 +636,10 @@ storageDriverS3NewWrite(
ASSERT(this != NULL);
ASSERT(file != NULL);
ASSERT(modeFile == 0);
ASSERT(modePath == 0);
ASSERT(createPath);
THROW(AssertError, "NOT YET IMPLEMENTED");
FUNCTION_LOG_RETURN(STORAGE_FILE_WRITE, NULL);
FUNCTION_LOG_RETURN(
STORAGE_FILE_WRITE, storageDriverS3FileWriteInterface(storageDriverS3FileWriteNew(this, file, this->partSize)));
}
/***********************************************************************************************************************************

20
src/storage/driver/s3/storage.h
View File

@ -31,8 +31,8 @@ Constructor
StorageDriverS3 *storageDriverS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, const String *region, const String *accessKey, const String *secretAccessKey,
const String *securityToken, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile,
const String *caPath);
const String *securityToken, size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer,
const String *caFile, const String *caPath);
/***********************************************************************************************************************************
Functions
@ -49,7 +49,21 @@ void storageDriverS3PathRemove(StorageDriverS3 *this, const String *path, bool e
void storageDriverS3PathSync(StorageDriverS3 *this, const String *path, bool ignoreMissing);
void storageDriverS3Remove(StorageDriverS3 *this, const String *file, bool errorOnMissing);
Buffer *storageDriverS3Request(
/***********************************************************************************************************************************
Perform an S3 Request
***********************************************************************************************************************************/
#define FUNCTION_LOG_STORAGE_DRIVER_S3_REQUEST_RESULT_TYPE \
StorageDriverS3RequestResult
#define FUNCTION_LOG_STORAGE_DRIVER_S3_REQUEST_RESULT_FORMAT(value, buffer, bufferSize) \
objToLog(&value, "StorageDriverS3RequestResult", buffer, bufferSize)
typedef struct StorageDriverS3RequestResult
{
HttpHeader *responseHeader;
Buffer *response;
} StorageDriverS3RequestResult;
StorageDriverS3RequestResult storageDriverS3Request(
StorageDriverS3 *this, const String *verb, const String *uri, const HttpQuery *query, const Buffer *body, bool returnContent,
bool allowMissing);

2
src/storage/helper.c
View File

@ -257,7 +257,7 @@ storageRepoGet(const String *type, bool write)
cfgOptionStr(cfgOptRepoPath), write, storageRepoPathExpression, cfgOptionStr(cfgOptRepoS3Bucket),
cfgOptionStr(cfgOptRepoS3Endpoint), cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key),
cfgOptionStr(cfgOptRepoS3KeySecret), cfgOptionTest(cfgOptRepoS3Token) ? cfgOptionStr(cfgOptRepoS3Token) : NULL,
cfgOptionTest(cfgOptRepoS3Host) ? cfgOptionStr(cfgOptRepoS3Host) : NULL,
(size_t)5 * 1024 * 1024, cfgOptionTest(cfgOptRepoS3Host) ? cfgOptionStr(cfgOptRepoS3Host) : NULL,
STORAGE_DRIVER_S3_PORT_DEFAULT, STORAGE_DRIVER_S3_TIMEOUT_DEFAULT, cfgOptionBool(cfgOptRepoS3VerifySsl),
cfgOptionTest(cfgOptRepoS3CaFile) ? cfgOptionStr(cfgOptRepoS3CaFile) : NULL,
cfgOptionTest(cfgOptRepoS3CaPath) ? cfgOptionStr(cfgOptRepoS3CaPath) : NULL));

1
test/define.yaml
View File

@ -534,6 +534,7 @@ unit:
coverage:
storage/driver/s3/fileRead: full
storage/driver/s3/fileWrite: full
storage/driver/s3/storage: full
storage/helper: full
storage/storage: full

209
test/src/module/storage/s3Test.c
View File

@ -15,27 +15,54 @@ Test server
"????????????????????????????????????????????????????????????????"
static const char *
testS3ServerRequest(const char *verb, const char *uri)
testS3ServerRequest(const char *verb, const char *uri, const char *content)
{
String *request = strNewFmt(
"%s %s HTTP/1.1\r\n"
"authorization:AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/" DATE_REPLACE "/us-east-1/s3/aws4_request,"
"SignedHeaders=content-length;host;x-amz-content-sha256;x-amz-date,Signature=" SHA256_REPLACE "\r\n"
"content-length:%u\r\n"
"host:" TLS_TEST_HOST "\r\n"
"x-amz-content-sha256:%s\r\n"
"x-amz-date:" DATETIME_REPLACE "\r\n"
"\r\n",
verb, uri, (unsigned int)0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
"SignedHeaders=content-length;",
verb, uri);
if (content != NULL)
strCat(request, "content-md5;");
strCatFmt(
request,
"host;x-amz-content-sha256;x-amz-date,Signature=" SHA256_REPLACE "\r\n"
"content-length:%zu\r\n",
content == NULL ? 0 : strlen(content));
if (content != NULL)
{
char md5Hash[HASH_TYPE_MD5_SIZE_HEX];
encodeToStr(encodeBase64, bufPtr(cryptoHashOneStr(HASH_TYPE_MD5_STR, strNew(content))), HASH_TYPE_M5_SIZE, md5Hash);
strCatFmt(request, "content-md5:%s\r\n", md5Hash);
}
strCatFmt(
request,
"host:" TLS_TEST_HOST "\r\n"
"x-amz-content-sha256:%s\r\n"
"x-amz-date:" DATETIME_REPLACE "\r\n"
"\r\n",
content == NULL ?
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" :
strPtr(bufHex(cryptoHashOneStr(HASH_TYPE_SHA256_STR, strNew(content)))));
if (content != NULL)
strCat(request, content);
return strPtr(request);
}
static const char *
testS3ServerResponse(unsigned int code, const char *message, const char *content)
testS3ServerResponse(unsigned int code, const char *message, const char *header, const char *content)
{
String *response = strNewFmt("HTTP/1.1 %u %s\r\n", code, message);
if (header != NULL)
strCatFmt(response, "%s\r\n", header);
if (content != NULL)
{
strCatFmt(
@ -62,37 +89,97 @@ testS3Server(void)
// storageDriverS3NewRead() and StorageDriverS3FileRead
// -------------------------------------------------------------------------------------------------------------------------
// Ignore missing file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt"));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL));
// Error on missing file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt"));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL));
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL));
// Get file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "this is a sample file"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, "this is a sample file"));
// Throw non-404 error
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt"));
harnessTlsServerReply(testS3ServerResponse(303, "Some bad status", "CONTENT"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL));
harnessTlsServerReply(testS3ServerResponse(303, "Some bad status", NULL, "CONTENT"));
// storageDriverS3NewWrite() and StorageDriverS3FileWrite
// -------------------------------------------------------------------------------------------------------------------------
// File is written all at once
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// Zero-length file
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", ""));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// File is written in chunks with nothing left over on close
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL));
harnessTlsServerReply(testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
"<Bucket>bucket</Bucket>"
"<Key>file.txt</Key>"
"<UploadId>WxRt</UploadId>"
"</InitiateMultipartUploadResult>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", "1234567890123456"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "etag:WxRt1", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", "7890123456789012"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "eTag:WxRt2", NULL));
harnessTlsServerExpect(testS3ServerRequest(
HTTP_VERB_POST, "/file.txt?uploadId=WxRt",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<CompleteMultipartUpload>"
"<Part><PartNumber>1</PartNumber><ETag>WxRt1</ETag></Part>"
"<Part><PartNumber>2</PartNumber><ETag>WxRt2</ETag></Part>"
"</CompleteMultipartUpload>\n"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// File is written in chunks with something left over on close
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL));
harnessTlsServerReply(testS3ServerResponse(
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
"<Bucket>bucket</Bucket>"
"<Key>file.txt</Key>"
"<UploadId>RR55</UploadId>"
"</InitiateMultipartUploadResult>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", "1234567890123456"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "etag:RR551", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", "7890"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "eTag:RR552", NULL));
harnessTlsServerExpect(testS3ServerRequest(
HTTP_VERB_POST, "/file.txt?uploadId=RR55",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<CompleteMultipartUpload>"
"<Part><PartNumber>1</PartNumber><ETag>RR551</ETag></Part>"
"<Part><PartNumber>2</PartNumber><ETag>RR552</ETag></Part>"
"</CompleteMultipartUpload>\n"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// storageDriverExists()
// -------------------------------------------------------------------------------------------------------------------------
// File missing
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=BOGUS"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=BOGUS", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK",
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
"</ListBucketResult>"));
// File exists
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=subdir%2Ffile1.txt"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=subdir%2Ffile1.txt", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK",
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
@ -104,10 +191,10 @@ testS3Server(void)
"</ListBucketResult>"));
// File does not exist but files with same prefix do
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=subdir%2Ffile1.txt"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?list-type=2&prefix=subdir%2Ffile1.txt", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK",
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
@ -121,14 +208,14 @@ testS3Server(void)
// storageDriverList()
// -------------------------------------------------------------------------------------------------------------------------
// Throw error
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"));
harnessTlsServerReply(testS3ServerResponse(344, "Another bad status", NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerReply(testS3ServerResponse(344, "Another bad status", NULL, NULL));
// list a file/path in root
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK",
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
@ -140,10 +227,10 @@ testS3Server(void)
"</ListBucketResult>"));
// list a file in root with expression
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK",
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
@ -152,10 +239,11 @@ testS3Server(void)
"</ListBucketResult>"));
// list files with continuation
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK",
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <NextContinuationToken>1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=</NextContinuationToken>"
@ -174,10 +262,11 @@ testS3Server(void)
testS3ServerRequest(
HTTP_VERB_GET,
"/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2"
"&prefix=path%2Fto%2F"));
"&prefix=path%2Fto%2F",
NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK",
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
@ -189,10 +278,11 @@ testS3Server(void)
"</ListBucketResult>"));
// list files with expression
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest"));
harnessTlsServerExpect(
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest", NULL));
harnessTlsServerReply(
testS3ServerResponse(
200, "OK",
200, "OK", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
" <Contents>"
@ -308,7 +398,7 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
StorageDriverS3 *driver = storageDriverS3New(
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, NULL, 0, 0, true, NULL, NULL);
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, 16, NULL, 0, 0, true, NULL, NULL);
HttpHeader *header = httpHeaderNew(NULL);
@ -360,7 +450,8 @@ testRun(void)
// Test with security token
// -------------------------------------------------------------------------------------------------------------------------
driver = storageDriverS3New(
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, securityToken, NULL, 0, 0, true, NULL, NULL);
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, securityToken, 16, NULL, 0, 0, true, NULL,
NULL);
TEST_RESULT_VOID(
storageDriverS3Auth(
@ -376,12 +467,12 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("storageDriverS3*() and StorageDriverS3FileRead"))
if (testBegin("storageDriverS3*(), StorageDriverS3FileRead, and StorageDriverS3FileWrite"))
{
testS3Server();
StorageDriverS3 *s3Driver = storageDriverS3New(
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, host, port, 1000, true, NULL, NULL);
path, true, NULL, bucket, endPoint, region, accessKey, secretAccessKey, NULL, 16, host, port, 1000, true, NULL, NULL);
Storage *s3 = storageDriverS3Interface(s3Driver);
// Coverage for noop functions
@ -421,6 +512,45 @@ testRun(void)
"*** Response Content ***:\n"
"CONTENT")
// storageDriverS3NewWrite() and StorageDriverS3FileWrite
// -------------------------------------------------------------------------------------------------------------------------
// File is written all at once
StorageFileWrite *write = NULL;
TEST_ASSIGN(write, storageNewWriteNP(s3, strNew("file.txt")), "new write file");
TEST_RESULT_VOID(storagePutNP(write, bufNewStr(strNew("ABCD"))), "put file all at once");
TEST_RESULT_BOOL(storageFileWriteAtomic(write), true, "write is atomic");
TEST_RESULT_BOOL(storageFileWriteCreatePath(write), true, "path will be created");
TEST_RESULT_UINT(storageFileWriteModeFile(write), 0, "file mode is 0");
TEST_RESULT_UINT(storageFileWriteModePath(write), 0, "path mode is 0");
TEST_RESULT_STR(strPtr(storageFileWriteName(write)), "/file.txt", "check file name");
TEST_RESULT_BOOL(storageFileWriteSyncFile(write), true, "file is synced");
TEST_RESULT_BOOL(storageFileWriteSyncPath(write), true, "path is synced");
TEST_RESULT_VOID(
storageDriverS3FileWriteClose((StorageDriverS3FileWrite *)storageFileWriteFileDriver(write)),
"close file again");
TEST_RESULT_VOID(
storageDriverS3FileWriteFree((StorageDriverS3FileWrite *)storageFileWriteFileDriver(write)),
"free file");
TEST_RESULT_VOID(storageDriverS3FileWriteFree(NULL), "free null file");
// Zero-length file
TEST_ASSIGN(write, storageNewWriteNP(s3, strNew("file.txt")), "new write file");
TEST_RESULT_VOID(storagePutNP(write, NULL), "write zero-length file");
// File is written in chunks with nothing left over on close
TEST_ASSIGN(write, storageNewWriteNP(s3, strNew("file.txt")), "new write file");
TEST_RESULT_VOID(
storagePutNP(write, bufNewStr(strNew("12345678901234567890123456789012"))),
"write file in chunks -- nothing left on close");
// File is written in chunks with something left over on close
TEST_ASSIGN(write, storageNewWriteNP(s3, strNew("file.txt")), "new write file");
TEST_RESULT_VOID(
storagePutNP(write, bufNewStr(strNew("12345678901234567890"))),
"write file in chunks -- something left on close");
// storageDriverExists()
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_BOOL(storageExistsNP(s3, strNew("BOGUS")), false, "file does not exist");
@ -457,7 +587,6 @@ testRun(void)
// Coverage for unimplemented functions
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(storageInfoNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
TEST_ERROR(storageNewWriteNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
TEST_ERROR(storagePathRemoveNP(s3, strNew("path")), AssertError, "NOT YET IMPLEMENTED");
TEST_ERROR(storageRemoveNP(s3, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
}