db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-06-20 01:17:49 +02:00
Code Issues Releases Activity

HTTP support for S3, GCS, and Azure.

Browse Source 
Allow users to specify HTTP in the endpoint but default to using HTTPS in all other scenarios to preserve the existing behavior.

Extend HttpUrl with a `defaultType` parameter to support either:
- Explicitly specifying a protocol via `.type` and enforcing that protocol is used in the URL.
- Allowing protocol to be parsed from URL, but providing default via `.defaultType` if no protocol is found in the URL.

Add partial write handling in fdWrite() to support non-blocking socket operations. The write loop now handles EAGAIN errors by waiting for the file descriptor to become writable, and continues writing the remaining bytes when write() returns fewer bytes than requested. This is required for HTTP, which may use non-blocking sockets, but doesn't have built in handling like the TLS client we are using for HTTPS. Also wrap the write() call and add a shim and additional logging for easier unit testing.
This commit is contained in:
Will Morland
2025-12-29 10:39:21 +07:00
committed by GitHub
parent 1afaab1c1e
commit 2398dd4d43
24 changed files with 776 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/xml/release/2020s/2026/2.58.0.xml
+12
View File
@@ -19,6 +19,18 @@
</release-bug-list>
<release-feature-list>
<release-item>
<github-issue id="2340"/>
<github-pull-request id="2703"/>
<release-item-contributor-list>
<release-item-contributor id="will.morland"/>
<release-item-reviewer id="david.steele"/>
</release-item-contributor-list>
<p>HTTP support for <proper>S3</proper>, <proper>GCS</proper>, and <proper>Azure</proper>.</p>
</release-item>
<release-item>
<github-issue id="2666"/>
<github-pull-request id="2709"/>
doc/xml/release/contributor.xml
+5
View File
@@ -1206,6 +1206,11 @@
<contributor-name-display>Will M</contributor-name-display>
</contributor>
<contributor id="will.morland">
<contributor-name-display>Will Morland</contributor-name-display>
<contributor-id type="github">wjmorland</contributor-id>
</contributor>
<contributor id="william.cox">
<contributor-name-display>William Cox</contributor-name-display>
<contributor-id type="github">mydimension</contributor-id>
src/common/debug.h
+5
View File
@@ -273,6 +273,11 @@ FN_EXTERN size_t typeToLog(const char *typeName, char *buffer, size_t bufferSize
#define FUNCTION_LOG_STRINGZ_FORMAT(value, buffer, bufferSize) \
strzToLog(value, buffer, bufferSize)
#define FUNCTION_LOG_SSIZE_TYPE \
ssize_t
#define FUNCTION_LOG_SSIZE_FORMAT(value, buffer, bufferSize) \
cvtInt64ToZ(value, buffer, bufferSize)
/***********************************************************************************************************************************
Macros to return function results (or void)
***********************************************************************************************************************************/
src/common/io/fdWrite.c
+44 -2
View File
@@ -64,6 +64,22 @@ ioFdWriteReady(THIS_VOID, const bool error)
/***********************************************************************************************************************************
Write to the file descriptor
***********************************************************************************************************************************/
// Helper wrapper around write() system call for unit testing
static ssize_t
ioFdWriteInternal(const int fd, const void *const buffer, const size_t size)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INT, fd);
FUNCTION_TEST_PARAM_P(VOID, buffer);
FUNCTION_TEST_PARAM(SIZE, size);
FUNCTION_TEST_END();
ASSERT(fd >= 0);
ASSERT(buffer != NULL);
FUNCTION_TEST_RETURN(SSIZE, write(fd, buffer, size));
}
static void
ioFdWrite(THIS_VOID, const Buffer *const buffer)
{
@@ -77,8 +93,34 @@ ioFdWrite(THIS_VOID, const Buffer *const buffer)
ASSERT(this != NULL);
ASSERT(buffer != NULL);
THROW_ON_SYS_ERROR_FMT(
write(this->fd, bufPtrConst(buffer), bufUsed(buffer)) == -1, FileWriteError, "unable to write to %s", strZ(this->name));
// Handle partial writes and non-blocking socket operations
size_t totalWritten = 0;
size_t bufferRemaining = bufUsed(buffer);
while (bufferRemaining > 0)
{
const ssize_t result = ioFdWriteInternal(this->fd, bufPtrConst(buffer) + totalWritten, bufferRemaining);
if (result == -1)
{
// Handle non-blocking socket case where write buffer is full
if (errno == EAGAIN)
{
// Wait for socket to become writable (will throw on timeout)
ioFdWriteReady(this, true);
continue;
}
// Treat all other errors as fatal
THROW_SYS_ERROR_FMT(
FileWriteError, "unable to finish write to %s (wrote %zu/%zu bytes)", strZ(this->name), totalWritten,
bufUsed(buffer));
}
totalWritten += (size_t)result;
bufferRemaining -= (size_t)result;
}
FUNCTION_LOG_RETURN_VOID();
}
src/common/io/http/url.c
+4 -3
View File
@@ -56,6 +56,7 @@ httpUrlNewParse(const String *const url, const HttpUrlNewParseParam param)
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, url);
FUNCTION_TEST_PARAM(ENUM, param.type);
FUNCTION_TEST_PARAM(ENUM, param.defaultType);
FUNCTION_TEST_END();
ASSERT(url != NULL);
@@ -87,10 +88,10 @@ httpUrlNewParse(const String *const url, const HttpUrlNewParseParam param)
// If no protocol found then the first part is the host
if (this->pub.type == httpProtocolTypeAny)
{
// Protocol must be set explicitly
ASSERT(param.type != httpProtocolTypeAny);
// Must have either an explicitly set protocol via the type parameter or a default protocol if the type is any
ASSERT(param.type != httpProtocolTypeAny || param.defaultType != httpProtocolTypeAny);
this->pub.type = param.type;
this->pub.type = param.type != httpProtocolTypeAny ? param.type : param.defaultType;
}
// Else protocol was found
else
src/common/io/http/url.h
+1
View File
@@ -32,6 +32,7 @@ typedef struct HttpUrlNewParseParam
{
VAR_PARAM_HEADER;
HttpProtocolType type; // Expected protocol type (httpProtocolTypeAny if any)
HttpProtocolType defaultType; // Default protocol type if explicitly set type is any
} HttpUrlNewParseParam;
#define httpUrlNewParseP(url, ...) \
src/storage/azure/helper.c
+5 -3
View File
@@ -29,9 +29,10 @@ storageAzureHelper(const unsigned int repoIdx, const bool write, StoragePathExpr
{
// Parse the endpoint url
const HttpUrl *const url = httpUrlNewParseP(
cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx), .type = httpProtocolTypeHttps);
cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx), .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttps);
const String *endpoint = httpUrlHost(url);
unsigned int port = httpUrlPort(url);
HttpProtocolType protocolType = httpUrlProtocolType(url);
StorageAzureUriStyle uriStyle = (StorageAzureUriStyle)cfgOptionIdxStrId(cfgOptRepoAzureUriStyle, repoIdx);
@@ -41,10 +42,11 @@ storageAzureHelper(const unsigned int repoIdx, const bool write, StoragePathExpr
if (cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx) != NULL)
{
const HttpUrl *const url = httpUrlNewParseP(
cfgOptionIdxStr(cfgOptRepoStorageHost, repoIdx), .type = httpProtocolTypeHttps);
cfgOptionIdxStr(cfgOptRepoStorageHost, repoIdx), .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttps);
endpoint = httpUrlHost(url);
port = httpUrlPort(url);
protocolType = httpUrlProtocolType(url);
if (cfgOptionIdxSource(cfgOptRepoAzureUriStyle, repoIdx) == cfgSourceDefault)
uriStyle = storageAzureUriStylePath;
@@ -81,7 +83,7 @@ storageAzureHelper(const unsigned int repoIdx, const bool write, StoragePathExpr
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoTargetTime(), pathExpressionCallback,
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx), keyType, key,
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx),
cfgOptionIdxKvNull(cfgOptRepoStorageTag, repoIdx), endpoint, uriStyle, port, ioTimeoutMs(),
cfgOptionIdxKvNull(cfgOptRepoStorageTag, repoIdx), endpoint, uriStyle, port, ioTimeoutMs(), protocolType,
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
}
src/storage/azure/storage.c
+15 -6
View File
@@ -767,7 +767,8 @@ storageAzureNew(
const String *const path, const bool write, const time_t targetTime, StoragePathExpressionCallback pathExpressionFunction,
const String *const container, const String *const account, const StorageAzureKeyType keyType, const String *const key,
const size_t blockSize, const KeyValue *const tag, const String *const endpoint, const StorageAzureUriStyle uriStyle,
const unsigned int port, const TimeMSec timeout, const bool verifyPeer, const String *const caFile, const String *const caPath)
const unsigned int port, const TimeMSec timeout, const HttpProtocolType protocolType, const bool verifyPeer,
const String *const caFile, const String *const caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@@ -784,6 +785,7 @@ storageAzureNew(
FUNCTION_LOG_PARAM(ENUM, uriStyle);
FUNCTION_LOG_PARAM(UINT, port);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
FUNCTION_LOG_PARAM(ENUM, protocolType);
FUNCTION_LOG_PARAM(BOOL, verifyPeer);
FUNCTION_LOG_PARAM(STRING, caFile);
FUNCTION_LOG_PARAM(STRING, caPath);
@@ -824,12 +826,19 @@ storageAzureNew(
else
this->sasKey = httpQueryNewStr(key);
// Create the http client used to service requests
this->httpClient = httpClientNew(
tlsClientNewP(
// Create the http client used to service requests. Use plain socket for HTTP, TLS for HTTPS.
IoClient *ioClient;
if (protocolType == httpProtocolTypeHttp)
ioClient = sckClientNew(this->host, port, timeout, timeout);
else
{
ioClient = tlsClientNewP(
sckClientNew(this->host, port, timeout, timeout), this->host, timeout, timeout, verifyPeer, .caFile = caFile,
.caPath = caPath),
timeout);
.caPath = caPath);
}
this->httpClient = httpClientNew(ioClient, timeout);
// Create list of redacted headers
this->headerRedactList = strLstNew();
src/storage/azure/storage.h
+2 -1
View File
@@ -4,6 +4,7 @@ Azure Storage
#ifndef STORAGE_AZURE_STORAGE_H
#define STORAGE_AZURE_STORAGE_H
#include "common/io/http/url.h"
#include "storage/storage.h"
/***********************************************************************************************************************************
@@ -36,6 +37,6 @@ FN_EXTERN Storage *storageAzureNew(
const String *path, bool write, time_t targetTime, StoragePathExpressionCallback pathExpressionFunction,
const String *container, const String *account, StorageAzureKeyType keyType, const String *key, size_t blockSize,
const KeyValue *tag, const String *endpoint, StorageAzureUriStyle uriStyle, unsigned int port, TimeMSec timeout,
bool verifyPeer, const String *caFile, const String *caPath);
HttpProtocolType protocolType, bool verifyPeer, const String *caFile, const String *caPath);
#endif
src/storage/gcs/storage.c
+15 -7
View File
@@ -1292,16 +1292,24 @@ storageGcsNew(
break;
}
// Parse the endpoint to extract the host and port
const HttpUrl *const url = httpUrlNewParseP(endpoint, .type = httpProtocolTypeHttps);
// Parse the endpoint to extract the host, port, and protocol
const HttpUrl *const url = httpUrlNewParseP(endpoint, .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttps);
const HttpProtocolType protocolType = httpUrlProtocolType(url);
this->endpoint = httpUrlHost(url);
// Create the http client used to service requests
this->httpClient = httpClientNew(
tlsClientNewP(
// Create the http client used to service requests. Use plain socket for HTTP, TLS for HTTPS.
IoClient *ioClient;
if (protocolType == httpProtocolTypeHttp)
ioClient = sckClientNew(this->endpoint, httpUrlPort(url), timeout, timeout);
else
{
ioClient = tlsClientNewP(
sckClientNew(this->endpoint, httpUrlPort(url), timeout, timeout), this->endpoint, timeout, timeout, verifyPeer,
.caFile = caFile, .caPath = caPath),
timeout);
.caFile = caFile, .caPath = caPath);
}
this->httpClient = httpClientNew(ioClient, timeout);
// Create list of redacted headers
this->headerRedactList = strLstNew();
src/storage/s3/helper.c
+6 -3
View File
@@ -31,9 +31,11 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
MEM_CONTEXT_TEMP_BEGIN()
{
// Parse the endpoint url
const HttpUrl *const url = httpUrlNewParseP(cfgOptionIdxStr(cfgOptRepoS3Endpoint, repoIdx), .type = httpProtocolTypeHttps);
const HttpUrl *const url = httpUrlNewParseP(
cfgOptionIdxStr(cfgOptRepoS3Endpoint, repoIdx), .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttps);
const String *const endPoint = httpUrlHost(url);
unsigned int port = httpUrlPort(url);
HttpProtocolType protocolType = httpUrlProtocolType(url);
// If host was specified then use it
const String *host = NULL;
@@ -41,10 +43,11 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
if (cfgOptionIdxSource(cfgOptRepoStorageHost, repoIdx) != cfgSourceDefault)
{
const HttpUrl *const url = httpUrlNewParseP(
cfgOptionIdxStr(cfgOptRepoStorageHost, repoIdx), .type = httpProtocolTypeHttps);
cfgOptionIdxStr(cfgOptRepoStorageHost, repoIdx), .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttps);
host = httpUrlHost(url);
port = httpUrlPort(url);
protocolType = httpUrlProtocolType(url);
}
// If port was specified, overwrite the parsed/default port
@@ -89,7 +92,7 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3SseCustomerKey, repoIdx), role, webIdTokenFile,
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx),
cfgOptionIdxKvNull(cfgOptRepoStorageTag, repoIdx), host, port, ioTimeoutMs(),
cfgOptionIdxKvNull(cfgOptRepoStorageTag, repoIdx), host, port, ioTimeoutMs(), protocolType,
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx), cfgOptionIdxBool(cfgOptRepoS3RequesterPays, repoIdx));
}
src/storage/s3/storage.c
+15 -7
View File
@@ -1185,8 +1185,8 @@ storageS3New(
const StorageS3KeyType keyType, const String *const accessKey, const String *const secretAccessKey,
const String *const securityToken, const String *const kmsKeyId, const String *sseCustomerKey, const String *const credRole,
const String *const webIdTokenFile, const size_t partSize, const KeyValue *const tag, const String *host,
const unsigned int port, const TimeMSec timeout, const bool verifyPeer, const String *const caFile, const String *const caPath,
const bool requesterPays)
const unsigned int port, const TimeMSec timeout, const HttpProtocolType protocolType, const bool verifyPeer,
const String *const caFile, const String *const caPath, const bool requesterPays)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@@ -1210,6 +1210,7 @@ storageS3New(
FUNCTION_LOG_PARAM(STRING, host);
FUNCTION_LOG_PARAM(UINT, port);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
FUNCTION_LOG_PARAM(ENUM, protocolType);
FUNCTION_LOG_PARAM(BOOL, verifyPeer);
FUNCTION_LOG_PARAM(STRING, caFile);
FUNCTION_LOG_PARAM(STRING, caPath);
@@ -1251,14 +1252,21 @@ storageS3New(
httpQueryFree(query);
}
// Create the HTTP client used to service requests
// Create the http client used to service requests. Use plain socket for HTTP, TLS for HTTPS.
if (host == NULL)
host = this->bucketEndpoint;
this->httpClient = httpClientNew(
tlsClientNewP(
sckClientNew(host, port, timeout, timeout), host, timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
timeout);
IoClient *ioClient;
if (protocolType == httpProtocolTypeHttp)
ioClient = sckClientNew(host, port, timeout, timeout);
else
{
ioClient = tlsClientNewP(
sckClientNew(host, port, timeout, timeout), host, timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath);
}
this->httpClient = httpClientNew(ioClient, timeout);
// Initialize authentication
switch (this->keyType)
src/storage/s3/storage.h
+3 -1
View File
@@ -4,6 +4,7 @@ S3 Storage
#ifndef STORAGE_S3_STORAGE_H
#define STORAGE_S3_STORAGE_H
#include "common/io/http/url.h"
#include "storage/storage.h"
/***********************************************************************************************************************************
@@ -38,6 +39,7 @@ FN_EXTERN Storage *storageS3New(
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
const String *secretAccessKey, const String *securityToken, const String *kmsKeyId, const String *sseCustomerKey,
const String *credRole, const String *webIdTokenFile, size_t partSize, const KeyValue *tag, const String *host,
unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath, bool requesterPays);
unsigned int port, TimeMSec timeout, HttpProtocolType protocolType, bool verifyPeer, const String *caFile, const String *caPath,
bool requesterPays);
#endif
test/define.yaml
+6 -3
View File
@@ -298,6 +298,9 @@ unit:
common/io/fd:
function:
- fdReady
common/io/fdWrite:
function:
- ioFdWriteInternal
coverage:
- common/io/bufferRead
@@ -582,7 +585,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: azure
total: 3
total: 4
coverage:
- storage/azure/helper
@@ -597,7 +600,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: gcs
total: 3
total: 4
coverage:
- storage/gcs/helper
@@ -612,7 +615,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: s3
total: 2
total: 3
coverage:
- storage/s3/helper
test/src/common/harnessFd.c
+53 -2
View File
@@ -17,9 +17,13 @@ Shim install state
***********************************************************************************************************************************/
static struct
{
bool localShimFdReady; // Is shim installed?
bool localShimFdReadyOne; // Should the shim run once?
bool localShimFdReady; // Is fdReady shim installed?
bool localShimFdReadyOne; // Should the fdReady shim run once?
bool localShimFdReadyOneResult; // Shim result for single run
bool localShimIoFdWriteInternalOne; // Should ioFdWriteInternal shim run once?
ssize_t localShimIoFdWriteInternalOneResult; // Return value for single run
int localShimIoFdWriteInternalOneErrNo; // errno value for single run
} hrnFdStatic;
/***********************************************************************************************************************************
@@ -94,3 +98,50 @@ hrnFdReadyShimOne(const bool result)
FUNCTION_HARNESS_RETURN_VOID();
}
/***********************************************************************************************************************************
Shim ioFdWriteInternal()
***********************************************************************************************************************************/
static ssize_t
ioFdWriteInternal(const int fd, const void *const buffer, const size_t size)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(INT, fd);
FUNCTION_HARNESS_PARAM_P(VOID, buffer);
FUNCTION_HARNESS_PARAM(SIZE, size);
FUNCTION_HARNESS_END();
ssize_t result;
// If shim will run once then return the requested result
if (hrnFdStatic.localShimIoFdWriteInternalOne)
{
hrnFdStatic.localShimIoFdWriteInternalOne = false;
result = hrnFdStatic.localShimIoFdWriteInternalOneResult;
// Set errno if result is -1
if (result == -1)
errno = hrnFdStatic.localShimIoFdWriteInternalOneErrNo;
}
// Else call normal function
else
result = ioFdWriteInternal_SHIMMED(fd, buffer, size);
FUNCTION_HARNESS_RETURN(SSIZE, result);
}
/**********************************************************************************************************************************/
void
hrnIoFdWriteInternalShimOne(const ssize_t result, const int errNo)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(INT64, result);
FUNCTION_HARNESS_PARAM(INT, errNo);
FUNCTION_HARNESS_END();
hrnFdStatic.localShimIoFdWriteInternalOne = true;
hrnFdStatic.localShimIoFdWriteInternalOneResult = result;
hrnFdStatic.localShimIoFdWriteInternalOneErrNo = errNo;
FUNCTION_HARNESS_RETURN_VOID();
}
test/src/common/harnessFd.h
+3
View File
@@ -11,3 +11,6 @@ void hrnFdReadyShimUninstall(void);
// Use shim for one call
void hrnFdReadyShimOne(bool result);
// Use ioFdWriteInternal shim for one call - specify return value and errno
void hrnIoFdWriteInternalShimOne(ssize_t result, int errNo);
test/src/common/harnessHost.c
+2 -2
View File
@@ -739,7 +739,7 @@ hrnHostConfig(HrnHost *const this)
this->pub.repo1Storage = storageAzureNew(
hrnHostRepo1Path(this), true, 0, NULL, STRDEF(HRN_HOST_AZURE_CONTAINER), STRDEF(HRN_HOST_AZURE_ACCOUNT),
storageAzureKeyTypeShared, STRDEF(HRN_HOST_AZURE_KEY), 4 * 1024 * 1024, NULL, hrnHostIp(azure),
storageAzureUriStylePath, 443, ioTimeoutMs(), false, NULL, NULL);
storageAzureUriStylePath, 443, ioTimeoutMs(), httpProtocolTypeHttps, false, NULL, NULL);
}
MEM_CONTEXT_OBJ_END();
@@ -785,7 +785,7 @@ hrnHostConfig(HrnHost *const this)
hrnHostRepo1Path(this), true, 0, NULL, STRDEF(HRN_HOST_S3_BUCKET), STRDEF(HRN_HOST_S3_ENDPOINT),
storageS3UriStyleHost, STR(HRN_HOST_S3_REGION), storageS3KeyTypeShared, STRDEF(HRN_HOST_S3_ACCESS_KEY),
STRDEF(HRN_HOST_S3_ACCESS_SECRET_KEY), NULL, NULL, NULL, NULL, NULL, 5 * 1024 * 1024, NULL,
hrnHostIp(s3), 443, ioTimeoutMs(), false, NULL, NULL, NULL);
hrnHostIp(s3), 443, ioTimeoutMs(), httpProtocolTypeHttps, false, NULL, NULL, NULL);
}
MEM_CONTEXT_OBJ_END();
test/src/module/common/execTest.c
+3 -1
View File
@@ -102,7 +102,9 @@ testRun(void)
TEST_ERROR(ioReadLine(execIoRead(exec)), FileReadError, "unable to read from sleep read: [9] Bad file descriptor");
ioWriteStrLine(execIoWrite(exec), strNew());
TEST_ERROR(ioWriteFlush(execIoWrite(exec)), FileWriteError, "unable to write to sleep write: [9] Bad file descriptor");
TEST_ERROR(
ioWriteFlush(execIoWrite(exec)), FileWriteError,
"unable to finish write to sleep write (wrote 0/1 bytes): [9] Bad file descriptor");
sleepMSec(500);
TEST_RESULT_VOID(execFree(exec), "sleep exited as expected");
test/src/module/common/ioHttpTest.c
+43
View File
@@ -279,6 +279,49 @@ testRun(void)
TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(url, httpUrlToLog, logBuf, sizeof(logBuf)), "httpUrlToLog");
TEST_RESULT_Z(logBuf, "{https://test.com:443/}", "check log");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("URL with http:// + defaultType=https uses detected http");
TEST_ASSIGN(
url, httpUrlNewParseP(STRDEF("http://test.com"), .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttps),
"new");
TEST_RESULT_STR_Z(httpUrl(url), "http://test.com", "check url");
TEST_RESULT_STR_Z(httpUrlHost(url), "test.com", "check host");
TEST_RESULT_UINT(httpUrlPort(url), 80, "check port");
TEST_RESULT_UINT(httpUrlProtocolType(url), httpProtocolTypeHttp, "check protocol is http (detected)");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("URL with https:// + defaultType=http uses detected https");
TEST_ASSIGN(
url, httpUrlNewParseP(STRDEF("https://test.com"), .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttp),
"new");
TEST_RESULT_STR_Z(httpUrl(url), "https://test.com", "check url");
TEST_RESULT_STR_Z(httpUrlHost(url), "test.com", "check host");
TEST_RESULT_UINT(httpUrlPort(url), 443, "check port");
TEST_RESULT_UINT(httpUrlProtocolType(url), httpProtocolTypeHttps, "check protocol is https (detected)");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("URL with no protocol + defaultType=https uses default https");
TEST_ASSIGN(
url, httpUrlNewParseP(STRDEF("test.com:4443"), .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttps),
"new");
TEST_RESULT_STR_Z(httpUrl(url), "test.com:4443", "check url");
TEST_RESULT_STR_Z(httpUrlHost(url), "test.com", "check host");
TEST_RESULT_UINT(httpUrlPort(url), 4443, "check port");
TEST_RESULT_UINT(httpUrlProtocolType(url), httpProtocolTypeHttps, "check protocol is https (default)");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("URL with no protocol + defaultType=http uses default http");
TEST_ASSIGN(
url, httpUrlNewParseP(STRDEF("test.com"), .type = httpProtocolTypeAny, .defaultType = httpProtocolTypeHttp), "new");
TEST_RESULT_STR_Z(httpUrl(url), "test.com", "check url");
TEST_RESULT_STR_Z(httpUrlHost(url), "test.com", "check host");
TEST_RESULT_UINT(httpUrlPort(url), 80, "check port");
TEST_RESULT_UINT(httpUrlProtocolType(url), httpProtocolTypeHttp, "check protocol is http (default)");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("IPv6");
test/src/module/common/ioTest.c
+136
View File
@@ -783,6 +783,142 @@ testRun(void)
freeaddrinfo(hostBadAddress);
}
TRY_END();
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("partial write handling with pipes");
HRN_FORK_BEGIN()
{
HRN_FORK_CHILD_BEGIN()
{
IoWrite *write = ioFdWriteNewOpen(STRDEF("partial write test"), HRN_FORK_CHILD_WRITE_FD(), 2000);
// Write data that exercises the partial write loop
const Buffer *testBuffer = BUFSTRDEF("test data for partial writes");
TEST_RESULT_VOID(ioWrite(write, testBuffer), "write buffer with partial write handling");
ioWriteFlush(write);
}
HRN_FORK_CHILD_END();
HRN_FORK_PARENT_BEGIN()
{
IoRead *read = ioFdReadNewOpen(STRDEF("partial write test read"), HRN_FORK_PARENT_READ_FD(0), 2000);
Buffer *receiveBuffer = bufNew(1024);
TEST_RESULT_UINT(ioRead(read, receiveBuffer), 28, "received all bytes");
TEST_RESULT_STR_Z(strNewBuf(receiveBuffer), "test data for partial writes", "verify data integrity");
}
HRN_FORK_PARENT_END();
}
HRN_FORK_END();
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write error on closed pipe");
int pipeFds[2];
THROW_ON_SYS_ERROR(pipe(pipeFds) == -1, KernelError, "unable to create pipe");
// Close the read end to cause write errors
close(pipeFds[0]);
IoWrite *write = ioFdWriteNew(STRDEF("closed pipe"), pipeFds[1], 1000);
ioWriteOpen(write);
// Fill the pipe buffer first to ensure subsequent write will fail
Buffer *fillBuffer = bufNew(65536);
memset(bufPtr(fillBuffer), 'Z', 65536);
bufUsedSet(fillBuffer, 65536);
// Writing to a closed pipe should result in SIGPIPE/EPIPE error
// The error message will include the file descriptor name and byte counts
TRY_BEGIN()
{
ioWrite(write, fillBuffer);
TEST_RESULT_BOOL(false, true, "expected write to closed pipe to fail");
}
CATCH_ANY()
{
TEST_RESULT_BOOL(errorType() == &FileWriteError, true, "check error type");
TEST_RESULT_BOOL(strstr(errorMessage(), "unable to finish write to closed pipe") != NULL, true, "check error message");
}
TRY_END();
close(pipeFds[1]);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write with EAGAIN then successful retry");
HRN_FORK_BEGIN()
{
HRN_FORK_CHILD_BEGIN()
{
int fd = HRN_FORK_CHILD_WRITE_FD();
// Shim first fdWrite to return EAGAIN
hrnIoFdWriteInternalShimOne(-1, EAGAIN);
// Create IoFdWrite which should handle EAGAIN
IoWrite *writeFd = ioFdWriteNewOpen(STRDEF("EAGAIN test"), fd, 2000);
Buffer *buf = bufNew(1024);
memset(bufPtr(buf), 'T', 1024);
bufUsedSet(buf, 1024);
TEST_RESULT_VOID(ioWrite(writeFd, buf), "write handles EAGAIN");
ioWriteFlush(writeFd);
}
HRN_FORK_CHILD_END();
HRN_FORK_PARENT_BEGIN()
{
// Give child time to start
sleepMSec(50);
IoRead *readFd = ioFdReadNewOpen(STRDEF("EAGAIN read"), HRN_FORK_PARENT_READ_FD(0), 2000);
Buffer *recvBuf = bufNew(8192);
size_t total = 0;
while (true)
{
bufUsedZero(recvBuf);
bufLimitSet(recvBuf, bufSize(recvBuf));
size_t bytes = ioRead(readFd, recvBuf);
if (bytes == 0)
break;
total += bytes;
}
TEST_RESULT_BOOL(total == 1024, true, "received all data after EAGAIN");
}
HRN_FORK_PARENT_END();
}
HRN_FORK_END();
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write with EAGAIN then timeout");
// Shim first fdWrite to return EAGAIN
hrnIoFdWriteInternalShimOne(-1, EAGAIN);
// Shim fdReady to return false (simulate timeout after EAGAIN)
hrnFdReadyShimOne(false);
int pipeFd[2];
THROW_ON_SYS_ERROR(pipe(pipeFd) == -1, KernelError, "unable to create pipe");
IoWrite *writeFd = ioFdWriteNewOpen(STRDEF("EAGAIN timeout"), pipeFd[1], 10);
Buffer *buf = bufNew(1024);
memset(bufPtr(buf), 'Z', 1024);
bufUsedSet(buf, 1024);
TEST_ERROR(
ioWrite(writeFd, buf), FileWriteError,
"timeout after 10ms waiting for write to 'EAGAIN timeout'");
close(pipeFd[0]);
close(pipeFd[1]);
}
// *****************************************************************************************************************************
test/src/module/protocol/protocolTest.c
+1 -1
View File
@@ -343,7 +343,7 @@ testRun(void)
hrnLogReplaceAdd(" \\[10\\] No child process(es){0,1}", "process(es){0,1}", "processes", false);
TEST_RESULT_LOG(
"P00 WARN: unable to write to invalid: [9] Bad file descriptor\n"
"P00 WARN: unable to finish write to invalid (wrote 0/12 bytes): [9] Bad file descriptor\n"
"P00 WARN: unable to wait on child process: [10] No child [processes]");
}
test/src/module/storage/azureTest.c
+117 -66
View File
@@ -399,8 +399,8 @@ testRun(void)
(StorageAzure *)storageDriver(
storageAzureNew(
STRDEF("/repo"), false, 0, NULL, TEST_CONTAINER_STR, TEST_ACCOUNT_STR, storageAzureKeyTypeShared,
TEST_KEY_SHARED_STR, 16, NULL, STRDEF("blob.core.windows.net"), storageAzureUriStyleHost, 443, 1000, true, NULL,
NULL)),
TEST_KEY_SHARED_STR, 16, NULL, STRDEF("blob.core.windows.net"), storageAzureUriStyleHost, 443, 1000,
httpProtocolTypeHttps, true, NULL, NULL)),
"new azure storage - shared key");
// -------------------------------------------------------------------------------------------------------------------------
@@ -441,7 +441,8 @@ testRun(void)
(StorageAzure *)storageDriver(
storageAzureNew(
STRDEF("/repo"), false, 0, NULL, TEST_CONTAINER_STR, TEST_ACCOUNT_STR, storageAzureKeyTypeSas, TEST_KEY_SAS_STR,
16, NULL, STRDEF("blob.core.usgovcloudapi.net"), storageAzureUriStyleHost, 443, 1000, true, NULL, NULL)),
16, NULL, STRDEF("blob.core.usgovcloudapi.net"), storageAzureUriStyleHost, 443, 1000, httpProtocolTypeHttps,
true, NULL, NULL)),
"new azure storage - sas key");
query = httpQueryAdd(httpQueryNewP(), STRDEF("a"), STRDEF("b"));
@@ -454,7 +455,7 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("StorageAzure, StorageReadAzure, and StorageWriteAzure"))
if (testBegin("StorageAzure, StorageReadAzure, and StorageWriteAzure with HTTPS"))
{
HRN_FORK_BEGIN()
{
@@ -507,26 +508,6 @@ testRun(void)
TEST_RESULT_PTR(
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), NULL, "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on missing file");
testRequestP(service, HTTP_VERB_GET, "/file.txt");
testResponseP(service, .code = 404);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("file.txt"))), FileMissingError,
"unable to open missing file '/file.txt' for read");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with offset and limit");
testRequestP(service, HTTP_VERB_GET, "/file.txt", .range = "1-21");
testResponseP(service, .content = "this is a sample file");
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(storage, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(21)))),
"this is a sample file", "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with retry");
@@ -699,51 +680,9 @@ testRun(void)
TEST_ASSIGN(write, storageNewWriteP(storage, STRDEF("file.txt")), "new write");
TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("12345678901234567890123456789012")), "write");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with something left over on close");
// Stop writing tags
driver->tag = NULL;
testRequestP(
service, HTTP_VERB_PUT, "/file.txt?blockid=0AAAAAAACCCCCCCDx0000000&comp=block",
.content = "1234567890123456");
testResponseP(service);
testRequestP(
service, HTTP_VERB_PUT, "/file.txt?blockid=0AAAAAAACCCCCCCDx0000001&comp=block", .content = "7890");
testResponseP(service);
testRequestP(
service, HTTP_VERB_PUT, "/file.txt?comp=blocklist",
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<BlockList>"
"<Uncommitted>0AAAAAAACCCCCCCDx0000000</Uncommitted>"
"<Uncommitted>0AAAAAAACCCCCCCDx0000001</Uncommitted>"
"</BlockList>\n");
testResponseP(service);
// Check that block size is updated during write
ioBufferSizeSet(6);
TEST_ASSIGN(write, storageNewWriteP(storage, STRDEF("file.txt")), "new write");
ioWriteOpen(storageWriteIo(write));
ioWrite(storageWriteIo(write), BUFSTRDEF("123456789012345678"));
TEST_RESULT_VOID(
bufResize(((StorageWriteAzure *)ioWriteDriver(storageWriteIo(write)))->blockBuffer, 17),
"resize part buffer to 17");
ioWrite(storageWriteIo(write), BUFSTRDEF("90"));
TEST_RESULT_UINT(
((StorageWriteAzure *)ioWriteDriver(storageWriteIo(write)))->blockSize, 16,
"part buffer reset to 16 (default)");
ioWriteClose(storageWriteIo(write));
ioBufferSizeSet(ioBufferSizeDefault);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("info for / does not exist");
@@ -1205,5 +1144,117 @@ testRun(void)
HRN_FORK_END();
}
// *****************************************************************************************************************************
if (testBegin("StorageAzure, StorageReadAzure, and StorageWriteAzure with HTTPS"))
{
HRN_FORK_BEGIN()
{
const unsigned int testPort = hrnServerPortNext();
HRN_FORK_CHILD_BEGIN(.prefix = "azure http server", .timeout = 5000)
{
TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolSocket, testPort), "azure http server");
}
HRN_FORK_CHILD_END();
HRN_FORK_PARENT_BEGIN()
{
IoWrite *service = hrnServerScriptBegin(
ioFdWriteNewOpen(STRDEF("azure http client"), HRN_FORK_PARENT_WRITE_FD(0), 2000));
StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE);
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/");
hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER);
hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "http://%s:%u", strZ(hrnServerHost()), testPort);
hrnCfgArgRawBool(argList, cfgOptRepoStorageVerifyTls, false);
hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT);
hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED);
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
Storage *storage = NULL;
TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage");
// Tests need the block size to be 16
driver = (StorageAzure *)storageDriver(storage);
driver->blockSize = 16;
// Test needs a predictable file id
driver->fileId = 0x0AAAAAAACCCCCCCD;
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on missing file");
hrnServerScriptAccept(service);
testRequestP(service, HTTP_VERB_GET, "/file.txt");
testResponseP(service, .code = 404);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("file.txt"))), FileMissingError,
"unable to open missing file '/file.txt' for read");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with offset and limit");
testRequestP(service, HTTP_VERB_GET, "/file.txt", .range = "1-21");
testResponseP(service, .content = "this is a sample file");
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(storage, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(21)))),
"this is a sample file", "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with something left over on close");
testRequestP(
service, HTTP_VERB_PUT, "/file.txt?blockid=0AAAAAAACCCCCCCDx0000000&comp=block",
.content = "1234567890123456");
testResponseP(service);
testRequestP(
service, HTTP_VERB_PUT, "/file.txt?blockid=0AAAAAAACCCCCCCDx0000001&comp=block", .content = "7890");
testResponseP(service);
testRequestP(
service, HTTP_VERB_PUT, "/file.txt?comp=blocklist",
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<BlockList>"
"<Uncommitted>0AAAAAAACCCCCCCDx0000000</Uncommitted>"
"<Uncommitted>0AAAAAAACCCCCCCDx0000001</Uncommitted>"
"</BlockList>\n");
testResponseP(service);
// Check that block size is updated during write
const size_t ioBufferSizeDefault = ioBufferSize();
ioBufferSizeSet(6);
StorageWrite *write = NULL;
TEST_ASSIGN(write, storageNewWriteP(storage, STRDEF("file.txt")), "new write");
ioWriteOpen(storageWriteIo(write));
ioWrite(storageWriteIo(write), BUFSTRDEF("123456789012345678"));
TEST_RESULT_VOID(
bufResize(((StorageWriteAzure *)ioWriteDriver(storageWriteIo(write)))->blockBuffer, 17),
"resize part buffer to 17");
ioWrite(storageWriteIo(write), BUFSTRDEF("90"));
TEST_RESULT_UINT(
((StorageWriteAzure *)ioWriteDriver(storageWriteIo(write)))->blockSize, 16,
"part buffer reset to 16 (default)");
ioWriteClose(storageWriteIo(write));
ioBufferSizeSet(ioBufferSizeDefault);
hrnServerScriptEnd(service);
}
HRN_FORK_PARENT_END();
}
HRN_FORK_END();
}
FUNCTION_HARNESS_RETURN_VOID();
}
test/src/module/storage/gcsTest.c
+163 -81
View File
@@ -63,6 +63,32 @@ STRING_STATIC(TEST_TOKEN_STR, TEST_TOKEN);
"}\n"
// {uncrustify_on}
/***********************************************************************************************************************************
Helper to generate auth request. The JWT part will need to be ? since it can vary in content and size.
***********************************************************************************************************************************/
static String *
testAuthRequest(const Storage *const storage)
{
const char *const preamble = "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=";
const String *const jwt = storageGcsAuthJwt(((StorageGcs *)storageDriver(storage)), time(NULL));
String *const result = strCatFmt(
strNew(),
"POST /token HTTP/1.1\r\n"
"user-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n"
"content-length:%zu\r\n"
"content-type:application/x-www-form-urlencoded\r\n"
"host:%s\r\n"
"\r\n"
"%s",
strSize(jwt) + strlen(preamble), strZ(hrnServerHost()), preamble);
for (unsigned int jwtIdx = 0; jwtIdx < strSize(jwt); jwtIdx++)
strCatChr(result, '?');
return result;
}
/***********************************************************************************************************************************
Helper to build test requests
***********************************************************************************************************************************/
@@ -301,7 +327,7 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("StorageGcs, StorageReadGcs, and StorageWriteGcs"))
if (testBegin("StorageGcs, StorageReadGcs, and StorageWriteGcs with HTTPS"))
{
HRN_FORK_BEGIN()
{
@@ -348,7 +374,7 @@ testRun(void)
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_GCS_TYPE);
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/");
hrnCfgArgRawZ(argList, cfgOptRepoGcsBucket, TEST_BUCKET);
hrnCfgArgRawFmt(argList, cfgOptRepoGcsEndpoint, "%s:%u", strZ(hrnServerHost()), testPort);
hrnCfgArgRawFmt(argList, cfgOptRepoGcsEndpoint, "https://%s:%u", strZ(hrnServerHost()), testPort);
hrnCfgArgRawBool(argList, cfgOptRepoStorageVerifyTls, TEST_IN_CONTAINER);
hrnCfgEnvRawZ(cfgOptRepoGcsKey, TEST_KEY_FILE);
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
@@ -357,23 +383,8 @@ testRun(void)
Storage *storage = NULL;
TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage");
// Generate the auth request. The JWT part will need to be ? since it can vary in content and size.
const char *const preamble = "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=";
const String *const jwt = storageGcsAuthJwt(((StorageGcs *)storageDriver(storage)), time(NULL));
String *const authRequest = strCatFmt(
strNew(),
"POST /token HTTP/1.1\r\n"
"user-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n"
"content-length:%zu\r\n"
"content-type:application/x-www-form-urlencoded\r\n"
"host:%s\r\n"
"\r\n"
"%s",
strSize(jwt) + strlen(preamble), strZ(hrnServerHost()), preamble);
for (unsigned int jwtIdx = 0; jwtIdx < strSize(jwt); jwtIdx++)
strCatChr(authRequest, '?');
// Generate the auth request
const String *const authRequest = testAuthRequest(storage);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("create bucket");
@@ -426,26 +437,6 @@ testRun(void)
TEST_RESULT_PTR(
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), NULL, "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on missing file");
testRequestP(service, HTTP_VERB_GET, .object = "file.txt", .query = "alt=media");
testResponseP(service, .code = 404);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("file.txt"))), FileMissingError,
"unable to open missing file '/file.txt' for read");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with offset and limit");
testRequestP(service, HTTP_VERB_GET, .object = "file.txt", .query = "alt=media", .range = "1-21");
testResponseP(service, .content = "this is a sample file");
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(storage, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(21)))),
"this is a sample file", "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with retry");
@@ -703,48 +694,6 @@ testRun(void)
((StorageGcs *)storageDriver(storage))->tag = NULL;
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with something left over on close");
testRequestP(service, HTTP_VERB_POST, .upload = true, .query = "name=file.txt&uploadType=resumable");
testResponseP(service, .header = "x-guploader-uploadid:ulid2");
testRequestP(
service, HTTP_VERB_PUT, .upload = true, .noAuth = true,
.query = "name=file.txt&uploadType=resumable&upload_id=ulid2", .contentRange = "0-15/*",
.content = "1234567890123456");
testResponseP(service, .code = 503);
testRequestP(
service, HTTP_VERB_PUT, .upload = true, .noAuth = true,
.query = "name=file.txt&uploadType=resumable&upload_id=ulid2", .contentRange = "0-15/*",
.content = "1234567890123456");
testResponseP(service, .code = 308);
testRequestP(
service, HTTP_VERB_PUT, .upload = true, .noAuth = true,
.query = "fields=md5Hash%2Csize&name=file.txt&uploadType=resumable&upload_id=ulid2", .contentRange = "16-19/20",
.content = "7890");
testResponseP(service, .content = "{\"md5Hash\":\"/YXmLZvrRUKHcexohBiycQ==\",\"size\":\"20\"}");
// Check that chunk size is updated during write
ioBufferSizeSet(6);
TEST_ASSIGN(write, storageNewWriteP(storage, STRDEF("file.txt")), "new write");
ioWriteOpen(storageWriteIo(write));
ioWrite(storageWriteIo(write), BUFSTRDEF("123456789012345678"));
TEST_RESULT_VOID(
bufResize(((StorageWriteGcs *)ioWriteDriver(storageWriteIo(write)))->chunkBuffer, 17),
"resize part buffer to 17");
ioWrite(storageWriteIo(write), BUFSTRDEF("90"));
TEST_RESULT_UINT(
((StorageWriteGcs *)ioWriteDriver(storageWriteIo(write)))->chunkSize, 16, "part buffer reset to 16 (default)");
ioWriteClose(storageWriteIo(write));
ioBufferSizeSet(ioBufferSizeDefault);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on resumable upload (upload_id is redacted)");
@@ -1347,5 +1296,138 @@ testRun(void)
HRN_FORK_END();
}
// *****************************************************************************************************************************
if (testBegin("StorageGcs, StorageReadGcs, and StorageWriteGcs with HTTP"))
{
HRN_FORK_BEGIN()
{
const String *const testHost = hrnServerHost();
const unsigned int testPort = hrnServerPortNext();
const unsigned int testPortAuth = hrnServerPortNext();
HRN_STORAGE_PUT(storageTest, TEST_KEY_FILE, BUFSTR(strNewFmt(TEST_KEY, strZ(testHost), testPortAuth)));
HRN_FORK_CHILD_BEGIN(.prefix = "gcs http server", .timeout = 10000)
{
TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolSocket, testPort), "gcs http server");
}
HRN_FORK_CHILD_END();
HRN_FORK_CHILD_BEGIN(.prefix = "auth server", .timeout = 10000)
{
TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolTls, testPortAuth), "auth server");
}
HRN_FORK_CHILD_END();
HRN_FORK_PARENT_BEGIN()
{
IoWrite *service = hrnServerScriptBegin(
ioFdWriteNewOpen(STRDEF("gcs http client write"), HRN_FORK_PARENT_WRITE_FD(0), 2000));
IoWrite *auth = hrnServerScriptBegin(
ioFdWriteNewOpen(STRDEF("auth http client write"), HRN_FORK_PARENT_WRITE_FD(1), 2000));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("test service auth");
StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_GCS_TYPE);
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/");
hrnCfgArgRawZ(argList, cfgOptRepoGcsBucket, TEST_BUCKET);
hrnCfgArgRawFmt(argList, cfgOptRepoGcsEndpoint, "http://%s:%u", strZ(hrnServerHost()), testPort);
hrnCfgArgRawBool(argList, cfgOptRepoStorageVerifyTls, TEST_IN_CONTAINER);
hrnCfgEnvRawZ(cfgOptRepoGcsKey, TEST_KEY_FILE);
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
hrnCfgEnvRemoveRaw(cfgOptRepoGcsKey);
Storage *storage = NULL;
TEST_ASSIGN(storage, storageRepoGet(0, true), "get repo storage");
// Tests need the chunk size to be 16
((StorageGcs *)storageDriver(storage))->chunkSize = 16;
// Generate the auth request
const String *const authRequest = testAuthRequest(storage);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on missing file");
hrnServerScriptAccept(auth);
hrnServerScriptExpect(auth, authRequest);
testResponseP(auth, .content = "{\"access_token\":\"X\",\"token_type\":\"X\",\"expires_in\":7200}");
hrnServerScriptClose(auth);
hrnServerScriptEnd(auth);
hrnServerScriptAccept(service);
testRequestP(service, HTTP_VERB_GET, .object = "file.txt", .query = "alt=media");
testResponseP(service, .code = 404);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("file.txt"))), FileMissingError,
"unable to open missing file '/file.txt' for read");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with offset and limit");
testRequestP(service, HTTP_VERB_GET, .object = "file.txt", .query = "alt=media", .range = "1-21");
testResponseP(service, .content = "this is a sample file");
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(storage, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(21)))),
"this is a sample file", "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with something left over on close");
testRequestP(service, HTTP_VERB_POST, .upload = true, .query = "name=file.txt&uploadType=resumable");
testResponseP(service, .header = "x-guploader-uploadid:ulid2");
testRequestP(
service, HTTP_VERB_PUT, .upload = true, .noAuth = true,
.query = "name=file.txt&uploadType=resumable&upload_id=ulid2", .contentRange = "0-15/*",
.content = "1234567890123456");
testResponseP(service, .code = 503);
testRequestP(
service, HTTP_VERB_PUT, .upload = true, .noAuth = true,
.query = "name=file.txt&uploadType=resumable&upload_id=ulid2", .contentRange = "0-15/*",
.content = "1234567890123456");
testResponseP(service, .code = 308);
testRequestP(
service, HTTP_VERB_PUT, .upload = true, .noAuth = true,
.query = "fields=md5Hash%2Csize&name=file.txt&uploadType=resumable&upload_id=ulid2", .contentRange = "16-19/20",
.content = "7890");
testResponseP(service, .content = "{\"md5Hash\":\"/YXmLZvrRUKHcexohBiycQ==\",\"size\":\"20\"}");
// Check that chunk size is updated during write
const size_t ioBufferSizeDefault = ioBufferSize();
ioBufferSizeSet(6);
StorageWrite *write = NULL;
TEST_ASSIGN(write, storageNewWriteP(storage, STRDEF("file.txt")), "new write");
ioWriteOpen(storageWriteIo(write));
ioWrite(storageWriteIo(write), BUFSTRDEF("123456789012345678"));
TEST_RESULT_VOID(
bufResize(((StorageWriteGcs *)ioWriteDriver(storageWriteIo(write)))->chunkBuffer, 17),
"resize part buffer to 17");
ioWrite(storageWriteIo(write), BUFSTRDEF("90"));
TEST_RESULT_UINT(
((StorageWriteGcs *)ioWriteDriver(storageWriteIo(write)))->chunkSize, 16, "part buffer reset to 16 (default)");
ioWriteClose(storageWriteIo(write));
ioBufferSizeSet(ioBufferSizeDefault);
hrnServerScriptEnd(service);
}
HRN_FORK_PARENT_END();
}
HRN_FORK_END();
}
FUNCTION_HARNESS_RETURN_VOID();
}
test/src/module/storage/s3Test.c
+117 -79
View File
@@ -465,7 +465,7 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("storageS3*(), StorageReadS3, and StorageWriteS3"))
if (testBegin("StorageS3, StorageReadS3, and StorageWriteS3 with HTTPS"))
{
HRN_FORK_BEGIN()
{
@@ -496,7 +496,7 @@ testRun(void)
TEST_TITLE("config with keys, token, and host with custom port");
StringList *argList = strLstDup(commonArgList);
hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), testPort);
hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "https://%s:%u", strZ(host), testPort);
hrnCfgEnvRaw(cfgOptRepoS3Token, securityToken);
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
@@ -543,26 +543,6 @@ testRun(void)
TEST_RESULT_PTR(storageGetP(storageNewReadP(s3, STRDEF("fi&le.txt"), .ignoreMissing = true)), NULL, "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on missing file");
testRequestP(service, s3, HTTP_VERB_GET, "/file.txt");
testResponseP(service, .code = 404);
TEST_ERROR(
storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), FileMissingError,
"unable to open missing file '/file.txt' for read");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with offset and limit");
testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "1-21");
testResponseP(service, .content = "this is a sample file");
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(21)))),
"this is a sample file", "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with retry");
@@ -1031,63 +1011,6 @@ testRun(void)
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with something left over on close");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=", .kms = "kmskey1", .sseC = "rA1P");
testResponseP(
service,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
"<Bucket>bucket</Bucket>"
"<Key>file.txt</Key>"
"<UploadId>RR55</UploadId>"
"</InitiateMultipartUploadResult>");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456",
.sseC = "rA1P");
testResponseP(service, .header = "etag:RR551");
testRequestP(
service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890",
.sseC = "rA1P");
testResponseP(service, .header = "eTag:RR552");
testRequestP(
service, s3, HTTP_VERB_POST, "/file.txt?uploadId=RR55",
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<CompleteMultipartUpload>"
"<Part><PartNumber>1</PartNumber><ETag>RR551</ETag></Part>"
"<Part><PartNumber>2</PartNumber><ETag>RR552</ETag></Part>"
"</CompleteMultipartUpload>\n");
testResponseP(
service,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<CompleteMultipartUploadResult><ETag>XXX</ETag></CompleteMultipartUploadResult>");
// Check that block size is updated during write
ioBufferSizeSet(6);
TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
ioWriteOpen(storageWriteIo(write));
ioWrite(storageWriteIo(write), BUFSTRDEF("123456789012345678"));
TEST_RESULT_VOID(
bufResize(((StorageWriteS3 *)ioWriteDriver(storageWriteIo(write)))->partBuffer, 17),
"resize part buffer to 17");
ioWrite(storageWriteIo(write), BUFSTRDEF("90"));
TEST_RESULT_UINT(
((StorageWriteS3 *)ioWriteDriver(storageWriteIo(write)))->partSize, 16, "part buffer reset to 16 (default)");
ioWriteClose(storageWriteIo(write));
ioBufferSizeSet(ioBufferSizeDefault);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("file missing");
@@ -1749,5 +1672,120 @@ testRun(void)
HRN_FORK_END();
}
// *****************************************************************************************************************************
if (testBegin("StorageS3, StorageReadS3, and StorageWriteS3 with HTTP"))
{
// -------------------------------------------------------------------------------------------------------------------------
HRN_FORK_BEGIN()
{
const unsigned int testPort = hrnServerPortNext();
HRN_FORK_CHILD_BEGIN(.prefix = "s3 http server", .timeout = 5000)
{
TEST_RESULT_VOID(
hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolSocket, testPort),
"s3 http server");
}
HRN_FORK_CHILD_END();
HRN_FORK_PARENT_BEGIN()
{
IoWrite *service = hrnServerScriptBegin(
ioFdWriteNewOpen(STRDEF("s3 http client"), HRN_FORK_PARENT_WRITE_FD(0), 2000));
StringList *argList = strLstDup(commonArgList);
hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "http://%s:%u", strZ(host), testPort);
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
Storage *s3 = storageRepoGet(0, true);
StorageS3 *driver = (StorageS3 *)storageDriver(s3);
// Set partSize to a small value for testing
driver->partSize = 16;
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on missing file");
hrnServerScriptAccept(service);
testRequestP(service, s3, HTTP_VERB_GET, "/file.txt");
testResponseP(service, .code = 404);
TEST_ERROR(
storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), FileMissingError,
"unable to open missing file '/file.txt' for read");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("get file with offset and limit");
testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .range = "1-21");
testResponseP(service, .content = "this is a sample file");
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("file.txt"), .offset = 1, .limit = VARUINT64(21)))),
"this is a sample file", "get file");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file in chunks with something left over on close");
testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=");
testResponseP(
service,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
"<Bucket>bucket</Bucket>"
"<Key>file.txt</Key>"
"<UploadId>RR55</UploadId>"
"</InitiateMultipartUploadResult>");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456");
testResponseP(service, .header = "etag:RR551");
testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890");
testResponseP(service, .header = "eTag:RR552");
testRequestP(
service, s3, HTTP_VERB_POST, "/file.txt?uploadId=RR55",
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<CompleteMultipartUpload>"
"<Part><PartNumber>1</PartNumber><ETag>RR551</ETag></Part>"
"<Part><PartNumber>2</PartNumber><ETag>RR552</ETag></Part>"
"</CompleteMultipartUpload>\n");
testResponseP(
service,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<CompleteMultipartUploadResult><ETag>XXX</ETag></CompleteMultipartUploadResult>");
// Check that block size is updated during write
const size_t ioBufferSizeDefault = ioBufferSize();
ioBufferSizeSet(6);
StorageWrite *write = NULL;
TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
ioWriteOpen(storageWriteIo(write));
ioWrite(storageWriteIo(write), BUFSTRDEF("123456789012345678"));
TEST_RESULT_VOID(
bufResize(((StorageWriteS3 *)ioWriteDriver(storageWriteIo(write)))->partBuffer, 17),
"resize part buffer to 17");
ioWrite(storageWriteIo(write), BUFSTRDEF("90"));
TEST_RESULT_UINT(
((StorageWriteS3 *)ioWriteDriver(storageWriteIo(write)))->partSize, 16, "part buffer reset to 16 (default)");
ioWriteClose(storageWriteIo(write));
ioBufferSizeSet(ioBufferSizeDefault);
hrnServerScriptEnd(service);
}
HRN_FORK_PARENT_END();
}
HRN_FORK_END();
}
FUNCTION_HARNESS_RETURN_VOID();
}