db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-30 05:39:12 +02:00
Code Issues Releases Activity

Add GCS batch delete support.

Browse Source 
The GCS driver sent a single file delete request for each file while deleting a path. Depending on latency this could lead to rather long delete times, especially noticeable during expiration.

Improve GCS delete to use batches, which require multipart HTTP, so also add multipart HTTP infrastructure.
This commit is contained in:
David Steele 2024-04-27 15:42:10 +10:00 committed by GitHub
parent e00e33b528
commit 76bcb740b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 1159 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/release/2024/2.52.xml
View File

@ -5,6 +5,17 @@
</text>
<release-feature-list>
<release-item>
<github-pull-request id="2326"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="reid.thompson"/>
</release-item-contributor-list>
<p>Add <proper>GCS</proper> batch delete support.</p>
</release-item>
<release-item>
<github-issue id="2279"/>
<github-pull-request id="2282"/>

249
src/common/io/http/request.c
View File

@ -25,11 +25,13 @@ STRING_EXTERN(HTTP_VERB_POST_STR, HTTP_VERB_PO
STRING_EXTERN(HTTP_VERB_PUT_STR, HTTP_VERB_PUT);
STRING_EXTERN(HTTP_HEADER_AUTHORIZATION_STR, HTTP_HEADER_AUTHORIZATION);
STRING_EXTERN(HTTP_HEADER_CONTENT_ID_STR, HTTP_HEADER_CONTENT_ID);
STRING_EXTERN(HTTP_HEADER_CONTENT_LENGTH_STR, HTTP_HEADER_CONTENT_LENGTH);
STRING_EXTERN(HTTP_HEADER_CONTENT_MD5_STR, HTTP_HEADER_CONTENT_MD5);
STRING_EXTERN(HTTP_HEADER_CONTENT_RANGE_STR, HTTP_HEADER_CONTENT_RANGE);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_STR, HTTP_HEADER_CONTENT_TYPE);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL_STR, HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_HTTP_STR, HTTP_HEADER_CONTENT_TYPE_HTTP);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_JSON_STR, HTTP_HEADER_CONTENT_TYPE_JSON);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_XML_STR, HTTP_HEADER_CONTENT_TYPE_XML);
STRING_EXTERN(HTTP_HEADER_ETAG_STR, HTTP_HEADER_ETAG);
@ -39,9 +41,6 @@ STRING_EXTERN(HTTP_HEADER_LAST_MODIFIED_STR, HTTP_HEADER_
STRING_EXTERN(HTTP_HEADER_RANGE_STR, HTTP_HEADER_RANGE);
#define HTTP_HEADER_USER_AGENT "user-agent"
// 5xx errors that should always be retried
#define HTTP_RESPONSE_CODE_RETRY_CLASS 5
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -54,6 +53,64 @@ struct HttpRequest
HttpSession *session; // Session for async requests
};
struct HttpRequestMulti
{
List *contentList; // Multipart content
size_t contentSize; // Size of all content (excluding boundaries)
Buffer *boundaryRaw; // Multipart boundary (not yet formatted for output)
};
/***********************************************************************************************************************************
Format the request (excluding content)
***********************************************************************************************************************************/
static String *
httpRequestFmt(
const String *const verb, const String *const path, const HttpQuery *const query, const HttpHeader *const header,
const bool agent)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, verb);
FUNCTION_TEST_PARAM(STRING, path);
FUNCTION_TEST_PARAM(HTTP_QUERY, query);
FUNCTION_TEST_PARAM(HTTP_HEADER, header);
FUNCTION_TEST_PARAM(BOOL, agent);
FUNCTION_TEST_END();
ASSERT(verb != NULL);
ASSERT(path != NULL);
ASSERT(header != NULL);
String *const result = strNew();
MEM_CONTEXT_TEMP_BEGIN()
{
// Add verb, path, and query
strCatFmt(
result, "%s %s%s%s " HTTP_VERSION "\r\n", strZ(verb), strZ(path), query == NULL ? "" : "?",
query == NULL ? "" : strZ(httpQueryRenderP(query)));
// Add user agent
if (agent)
strCatZ(result, HTTP_HEADER_USER_AGENT ":" PROJECT_NAME "/" PROJECT_VERSION "\r\n");
// Add headers
StringList *const headerList = httpHeaderList(header);
for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
{
const String *const headerKey = strLstGet(headerList, headerIdx);
strCatFmt(result, "%s:%s\r\n", strZ(headerKey), strZ(httpHeaderGet(header, headerKey)));
}
// Add blank line to end of headers
strCat(result, CRLF_STR);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN(STRING, result);
}
/***********************************************************************************************************************************
Process the request
***********************************************************************************************************************************/
@ -99,28 +156,13 @@ httpRequestProcess(HttpRequest *const this, const bool waitForResponse, const bo
{
session = httpClientOpen(this->client);
// Format the request and user agent
String *const requestStr =
strCatFmt(
strNew(),
"%s %s%s%s " HTTP_VERSION "\r\n" HTTP_HEADER_USER_AGENT ":" PROJECT_NAME "/" PROJECT_VERSION "\r\n",
strZ(httpRequestVerb(this)), strZ(httpRequestPath(this)), httpRequestQuery(this) == NULL ? "" : "?",
httpRequestQuery(this) == NULL ? "" : strZ(httpQueryRenderP(httpRequestQuery(this))));
// Add headers
const StringList *const headerList = httpHeaderList(httpRequestHeader(this));
for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
{
const String *const headerKey = strLstGet(headerList, headerIdx);
strCatFmt(
requestStr, "%s:%s\r\n", strZ(headerKey), strZ(httpHeaderGet(httpRequestHeader(this), headerKey)));
}
// Add blank line to end of headers and write the request as a buffer so secrets do not show up in logs
strCat(requestStr, CRLF_STR);
ioWrite(httpSessionIoWrite(session), BUFSTR(requestStr));
// Write the request as a buffer so secrets do not show up in logs
ioWrite(
httpSessionIoWrite(session),
BUFSTR(
httpRequestFmt(
httpRequestVerb(this), httpRequestPath(this), httpRequestQuery(this), httpRequestHeader(this),
true)));
// Write out content if any
if (this->content != NULL)
@ -142,7 +184,7 @@ httpRequestProcess(HttpRequest *const this, const bool waitForResponse, const bo
// Retry when response code is 5xx. These errors generally represent a server error for a request that looks
// valid. There are a few errors that might be permanently fatal but they are rare and it seems best not to
// try and pick and choose errors in this class to retry.
if (httpResponseCode(result) / 100 == HTTP_RESPONSE_CODE_RETRY_CLASS)
if (httpResponseCodeRetry(result))
THROW_FMT(ServiceError, "[%u] %s", httpResponseCode(result), strZ(httpResponseReason(result)));
// Move response to outer temp context
@ -302,6 +344,161 @@ httpRequestError(const HttpRequest *const this, HttpResponse *const response)
THROW(ProtocolError, strZ(error));
}
/**********************************************************************************************************************************/
#define HTTP_MULTIPART_BOUNDARY_INIT_SIZE (sizeof(HTTP_MULTIPART_BOUNDARY_INIT) - 1)
FN_EXTERN HttpRequestMulti *
httpRequestMultiNew(void)
{
FUNCTION_TEST_VOID();
OBJ_NEW_BEGIN(HttpRequestMulti, .childQty = MEM_CONTEXT_QTY_MAX)
{
*this = (HttpRequestMulti)
{
.boundaryRaw = bufNewC(HTTP_MULTIPART_BOUNDARY_INIT, HTTP_MULTIPART_BOUNDARY_INIT_SIZE),
.contentList = lstNewP(sizeof(Buffer *)),
};
}
OBJ_NEW_END();
FUNCTION_TEST_RETURN(HTTP_REQUEST_MULTI, this);
}
/**********************************************************************************************************************************/
#define HTTP_MULTIPART_BOUNDARY_EXTRA_SIZE (sizeof(HTTP_MULTIPART_BOUNDARY_EXTRA) - 1)
FN_EXTERN void
httpRequestMultiAdd(
HttpRequestMulti *const this, const String *const contentId, const String *const verb, const String *const path,
const HttpRequestNewParam param)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_REQUEST_MULTI, this);
FUNCTION_TEST_PARAM(STRING, contentId);
FUNCTION_TEST_PARAM(STRING, verb);
FUNCTION_TEST_PARAM(STRING, path);
FUNCTION_TEST_PARAM(HTTP_QUERY, param.query);
FUNCTION_TEST_PARAM(HTTP_HEADER, param.header);
FUNCTION_TEST_PARAM(BUFFER, param.content);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(contentId != NULL);
ASSERT(verb != NULL);
ASSERT(path != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Construct request header
String *const request = strNew();
strCatZ(request, HTTP_HEADER_CONTENT_TYPE ":" HTTP_HEADER_CONTENT_TYPE_HTTP "\r\n");
strCatZ(request, HTTP_HEADER_CONTENT_TRANSFER_ENCODING ":" HTTP_HEADER_CONTENT_TRANSFER_ENCODING_BINARY "\r\n");
strCatFmt(request, HTTP_HEADER_CONTENT_ID ":%s\r\n\r\n", strZ(contentId));
strCat(request, httpRequestFmt(verb, path, param.query, param.header, false));
MEM_CONTEXT_OBJ_BEGIN(this->contentList)
{
// Add content
Buffer *const content = bufNew(strSize(request) + (param.content == NULL ? 0 : bufUsed(param.content)));
bufCat(content, BUFSTR(request));
bufCat(content, param.content);
// Find a boundary that is not used in the content
while (bufFindP(content, this->boundaryRaw) != NULL)
{
if (bufUsed(this->boundaryRaw) == HTTP_MULTIPART_BOUNDARY_INIT_SIZE + HTTP_MULTIPART_BOUNDARY_EXTRA_SIZE)
THROW(AssertError, "unable to construct unique boundary");
bufCatC(
this->boundaryRaw, (unsigned char *)HTTP_MULTIPART_BOUNDARY_EXTRA,
bufUsed(this->boundaryRaw) - HTTP_MULTIPART_BOUNDARY_INIT_SIZE, HTTP_MULTIPART_BOUNDARY_NEXT);
}
// Add to list
lstAdd(this->contentList, &content);
this->contentSize += bufUsed(content);
}
MEM_CONTEXT_OBJ_END();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
#define HTTP_MULTIPART_BOUNDARY_PRE_SIZE (sizeof(HTTP_MULTIPART_BOUNDARY_PRE) - 1)
#define HTTP_MULTIPART_BOUNDARY_POST_SIZE (sizeof(HTTP_MULTIPART_BOUNDARY_POST) - 1)
#define HTTP_MULTIPART_BOUNDARY_POST_LAST_SIZE (sizeof(HTTP_MULTIPART_BOUNDARY_POST_LAST) - 1)
FN_EXTERN Buffer *
httpRequestMultiContent(HttpRequestMulti *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_REQUEST_MULTI, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(!lstEmpty(this->contentList));
const size_t boundarySize = bufUsed(this->boundaryRaw) + HTTP_MULTIPART_BOUNDARY_PRE_SIZE;
Buffer *const result = bufNew(
this->contentSize + (lstSize(this->contentList) * (boundarySize + HTTP_MULTIPART_BOUNDARY_POST_SIZE)) +
HTTP_MULTIPART_BOUNDARY_POST_LAST_SIZE);
MEM_CONTEXT_TEMP_BEGIN()
{
// Generate boundary
Buffer *const boundary = bufNew(boundarySize);
bufCat(boundary, BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_PRE));
bufCat(boundary, this->boundaryRaw);
// Add first boundary
bufCat(result, boundary);
bufCat(result, BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_POST));
// Add content and boundaries
for (unsigned int contentIdx = 0; contentIdx < lstSize(this->contentList); contentIdx++)
{
bufCat(result, *(Buffer **)lstGet(this->contentList, contentIdx));
bufCat(result, boundary);
if (contentIdx < lstSize(this->contentList) - 1)
bufCat(result, BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_POST));
else
bufCat(result, BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_POST_LAST));
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN(BUFFER, result);
}
/**********************************************************************************************************************************/
FN_EXTERN HttpHeader *
httpRequestMultiHeaderAdd(HttpRequestMulti *const this, HttpHeader *const header)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_REQUEST_MULTI, this);
FUNCTION_TEST_PARAM(HTTP_HEADER, header);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(header != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
httpHeaderAdd(
header, HTTP_HEADER_CONTENT_TYPE_STR,
strNewFmt(HTTP_HEADER_CONTENT_TYPE_MULTIPART "%s", strZ(strNewBuf(this->boundaryRaw))));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN(HTTP_HEADER, header);
}
/**********************************************************************************************************************************/
FN_EXTERN void
httpRequestToLog(const HttpRequest *const this, StringStatic *const debugLog)

53
src/common/io/http/request.h
View File

@ -12,6 +12,7 @@ behavior.
Object type
***********************************************************************************************************************************/
typedef struct HttpRequest HttpRequest;
typedef struct HttpRequestMulti HttpRequestMulti;
#include "common/io/http/header.h"
#include "common/io/http/query.h"
@ -39,18 +40,25 @@ STRING_DECLARE(HTTP_VERB_PUT_STR);
#define HTTP_HEADER_AUTHORIZATION "authorization"
STRING_DECLARE(HTTP_HEADER_AUTHORIZATION_STR);
#define HTTP_HEADER_CONTENT_ID "content-id"
STRING_DECLARE(HTTP_HEADER_CONTENT_ID_STR);
#define HTTP_HEADER_CONTENT_LENGTH "content-length"
STRING_DECLARE(HTTP_HEADER_CONTENT_LENGTH_STR);
#define HTTP_HEADER_CONTENT_MD5 "content-md5"
STRING_DECLARE(HTTP_HEADER_CONTENT_MD5_STR);
#define HTTP_HEADER_CONTENT_RANGE "content-range"
STRING_DECLARE(HTTP_HEADER_CONTENT_RANGE_STR);
#define HTTP_HEADER_CONTENT_TRANSFER_ENCODING "content-transfer-encoding"
#define HTTP_HEADER_CONTENT_TRANSFER_ENCODING_BINARY "binary"
#define HTTP_HEADER_CONTENT_TYPE "content-type"
STRING_DECLARE(HTTP_HEADER_CONTENT_TYPE_STR);
#define HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL "application/x-www-form-urlencoded"
STRING_DECLARE(HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL_STR);
#define HTTP_HEADER_CONTENT_TYPE_HTTP "application/http"
STRING_DECLARE(HTTP_HEADER_CONTENT_TYPE_HTTP_STR);
#define HTTP_HEADER_CONTENT_TYPE_JSON "application/json"
STRING_DECLARE(HTTP_HEADER_CONTENT_TYPE_JSON_STR);
#define HTTP_HEADER_CONTENT_TYPE_MULTIPART "multipart/mixed; boundary="
#define HTTP_HEADER_CONTENT_TYPE_XML "application/xml"
STRING_DECLARE(HTTP_HEADER_CONTENT_TYPE_XML_STR);
#define HTTP_HEADER_CONTENT_RANGE_BYTES "bytes"
@ -66,8 +74,15 @@ STRING_DECLARE(HTTP_HEADER_LAST_MODIFIED_STR);
STRING_DECLARE(HTTP_HEADER_RANGE_STR);
#define HTTP_HEADER_RANGE_BYTES "bytes"
#define HTTP_MULTIPART_BOUNDARY_INIT "QKX4EYg4"
#define HTTP_MULTIPART_BOUNDARY_NEXT 4
#define HTTP_MULTIPART_BOUNDARY_EXTRA "LARJ52gF4F239iNVrrC5w5aYskcGrWCXIFlMp5IxswggIhcX2A0gF9nrgN8q"
#define HTTP_MULTIPART_BOUNDARY_PRE "\r\n--"
#define HTTP_MULTIPART_BOUNDARY_POST "\r\n"
#define HTTP_MULTIPART_BOUNDARY_POST_LAST "--\r\n"
/***********************************************************************************************************************************
Constructors
Request Constructors
***********************************************************************************************************************************/
typedef struct HttpRequestNewParam
{
@ -83,7 +98,7 @@ typedef struct HttpRequestNewParam
FN_EXTERN HttpRequest *httpRequestNew(HttpClient *client, const String *verb, const String *path, HttpRequestNewParam param);
/***********************************************************************************************************************************
Getters/Setters
Request Getters/Setters
***********************************************************************************************************************************/
typedef struct HttpRequestPub
{
@ -122,7 +137,7 @@ httpRequestVerb(const HttpRequest *const this)
}
/***********************************************************************************************************************************
Functions
Request Functions
***********************************************************************************************************************************/
// Wait for a response from the request
FN_EXTERN HttpResponse *httpRequestResponse(HttpRequest *this, bool contentCache);
@ -138,7 +153,7 @@ httpRequestMove(HttpRequest *const this, MemContext *const parentNew)
}
/***********************************************************************************************************************************
Destructor
Request Destructor
***********************************************************************************************************************************/
FN_INLINE_ALWAYS void
httpRequestFree(HttpRequest *const this)
@ -146,14 +161,40 @@ httpRequestFree(HttpRequest *const this)
objFree(this);
}
/***********************************************************************************************************************************
Request Multi Constructors
***********************************************************************************************************************************/
FN_EXTERN HttpRequestMulti *httpRequestMultiNew(void);
/***********************************************************************************************************************************
Request Multi Functions
***********************************************************************************************************************************/
// Add a request to multipart content
#define httpRequestMultiAddP(this, contentId, verb, path, ...) \
httpRequestMultiAdd(this, contentId, verb, path, (HttpRequestNewParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN void httpRequestMultiAdd(
HttpRequestMulti *this, const String *contentId, const String *verb, const String *path, HttpRequestNewParam param);
// Concatenate multipart content
FN_EXTERN Buffer *httpRequestMultiContent(HttpRequestMulti *this);
// Add multipart header
FN_EXTERN HttpHeader *httpRequestMultiHeaderAdd(HttpRequestMulti *this, HttpHeader *header);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
FN_EXTERN void httpRequestToLog(const HttpRequest *this, StringStatic *debugLog);
#define FUNCTION_LOG_HTTP_REQUEST_TYPE \
#define FUNCTION_LOG_HTTP_REQUEST_TYPE \
HttpRequest *
#define FUNCTION_LOG_HTTP_REQUEST_FORMAT(value, buffer, bufferSize) \
#define FUNCTION_LOG_HTTP_REQUEST_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_OBJECT_FORMAT(value, httpRequestToLog, buffer, bufferSize)
#define FUNCTION_LOG_HTTP_REQUEST_MULTI_TYPE \
HttpRequestMulti *
#define FUNCTION_LOG_HTTP_REQUEST_MULTI_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "HttpRequestMulti", buffer, bufferSize)
#endif

457
src/common/io/http/response.c
View File

@ -4,6 +4,8 @@ HTTP Response
#include "build.auto.h"
#include "common/debug.h"
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "common/io/http/client.h"
#include "common/io/http/common.h"
#include "common/io/http/request.h"
@ -43,6 +45,13 @@ struct HttpResponse
Buffer *content; // Caches content once requested
};
struct HttpResponseMulti
{
const Buffer *content; // Response content
Buffer *boundary; // Multipart boundary
const unsigned char *boundaryLast; // Last boundary location
};
/***********************************************************************************************************************************
When response is done close/reuse the connection
***********************************************************************************************************************************/
@ -205,7 +214,160 @@ httpResponseEof(THIS_VOID)
FUNCTION_LOG_RETURN(BOOL, this->contentEof);
}
/***********************************************************************************************************************************
Read response status
***********************************************************************************************************************************/
static void
httpResponseStatusRead(HttpResponse *const this, IoRead *const read)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_RESPONSE, this);
FUNCTION_TEST_PARAM(IO_READ, read);
FUNCTION_TEST_END();
MEM_CONTEXT_TEMP_BEGIN()
{
// Read status
String *status = ioReadLine(read);
// Check status ends with a CR and remove it to make error formatting easier and more accurate
if (!strEndsWith(status, CR_STR))
THROW_FMT(FormatError, "HTTP response status '%s' should be CR-terminated", strZ(status));
status = strSubN(status, 0, strSize(status) - 1);
// Check status is at least the minimum required length to avoid harder to interpret errors later on
if (strSize(status) < sizeof(HTTP_VERSION) + 4)
THROW_FMT(FormatError, "HTTP response '%s' has invalid length", strZ(strTrim(status)));
// If HTTP/1.0 then the connection will be closed on content eof since connections are not reused by default
if (strBeginsWith(status, HTTP_VERSION_10_STR))
{
this->closeOnContentEof = true;
}
// Else check that the version is the default (1.1)
else if (!strBeginsWith(status, HTTP_VERSION_STR))
THROW_FMT(FormatError, "HTTP version of response '%s' must be " HTTP_VERSION " or " HTTP_VERSION_10, strZ(status));
// Read status code
status = strSub(status, sizeof(HTTP_VERSION));
int spacePos = strChr(status, ' ');
if (spacePos != 3)
THROW_FMT(FormatError, "response status '%s' must have a space after the status code", strZ(status));
this->pub.code = cvtZSubNToUInt(strZ(status), 0, (size_t)spacePos);
// Read reason phrase. A missing reason phrase will be represented as an empty string.
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pub.reason = strSub(status, (size_t)spacePos + 1);
}
MEM_CONTEXT_OBJ_END();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
Read response headers
***********************************************************************************************************************************/
static void
httpResponseHeaderRead(HttpResponse *const this, IoRead *const read)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_RESPONSE, this);
FUNCTION_TEST_PARAM(IO_READ, read);
FUNCTION_TEST_END();
MEM_CONTEXT_TEMP_BEGIN()
{
do
{
// Read the next header
String *const header = strTrim(ioReadLine(read));
// If the header is empty then we have reached the end of the headers
if (strSize(header) == 0)
break;
// Split the header and store it
int colonPos = strChr(header, ':');
if (colonPos < 0)
THROW_FMT(FormatError, "header '%s' missing colon", strZ(strTrim(header)));
const String *const headerKey = strLower(strTrim(strSubN(header, 0, (size_t)colonPos)));
String *const headerValue = strTrim(strSub(header, (size_t)colonPos + 1));
// Read transfer encoding (only chunked is supported)
if (strEq(headerKey, HTTP_HEADER_TRANSFER_ENCODING_STR))
{
// Error if transfer encoding is not chunked
if (!strEq(headerValue, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR))
{
THROW_FMT(
FormatError, "only '%s' is supported for '%s' header", HTTP_VALUE_TRANSFER_ENCODING_CHUNKED,
HTTP_HEADER_TRANSFER_ENCODING);
}
this->contentChunked = true;
}
// Read content size
if (strEq(headerKey, HTTP_HEADER_CONTENT_LENGTH_STR))
{
this->contentSize = cvtZToUInt64(strZ(headerValue));
this->contentRemaining = this->contentSize;
}
// If the server notified of a closed connection then close the client connection after reading content. This
// prevents doing a retry on the next request when using the closed connection.
if (strEq(headerKey, HTTP_HEADER_CONNECTION_STR) && strEq(strLower(headerValue), HTTP_VALUE_CONNECTION_CLOSE_STR))
this->closeOnContentEof = true;
// Add after header checks in case the value was modified
httpHeaderAdd(this->pub.header, headerKey, headerValue);
}
while (1);
// Error if transfer encoding and content length are both set
if (this->contentChunked && this->contentSize > 0)
{
THROW_FMT(
FormatError, "'%s' and '%s' headers are both set", HTTP_HEADER_TRANSFER_ENCODING,
HTTP_HEADER_CONTENT_LENGTH);
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
// Constructor helper
static HttpResponse *
httpResponseNewInternal(void)
{
FUNCTION_TEST_VOID();
OBJ_NEW_BEGIN(HttpResponse, .childQty = MEM_CONTEXT_QTY_MAX)
{
*this = (HttpResponse)
{
.pub =
{
.header = httpHeaderNew(NULL),
},
};
}
OBJ_NEW_END();
FUNCTION_TEST_RETURN(HTTP_RESPONSE, this);
}
FN_EXTERN HttpResponse *
httpResponseNew(HttpSession *const session, const String *const verb, const bool contentCache)
{
@ -218,148 +380,43 @@ httpResponseNew(HttpSession *const session, const String *const verb, const bool
ASSERT(session != NULL);
ASSERT(verb != NULL);
OBJ_NEW_BEGIN(HttpResponse, .childQty = MEM_CONTEXT_QTY_MAX)
HttpResponse *const this = httpResponseNewInternal();
this->session = httpSessionMove(session, objMemContext(this));
// Read status
httpResponseStatusRead(this, httpSessionIoReadP(this->session));
// Read headers
httpResponseHeaderRead(this, httpSessionIoReadP(this->session));
// Was content returned in the response? HEAD will report content but not actually return any.
this->contentExists =
(this->contentChunked || this->contentSize > 0 || this->closeOnContentEof) && !strEq(verb, HTTP_VERB_HEAD_STR);
this->contentEof = !this->contentExists;
// Create an io object, even if there is no content. This makes the logic for readers easier -- they can just check eof
// rather than also checking if the io object exists.
MEM_CONTEXT_OBJ_BEGIN(this)
{
*this = (HttpResponse)
{
.pub =
{
.header = httpHeaderNew(NULL),
},
.session = httpSessionMove(session, memContextCurrent()),
};
MEM_CONTEXT_TEMP_BEGIN()
{
// Read status
String *status = ioReadLine(httpSessionIoReadP(this->session));
// Check status ends with a CR and remove it to make error formatting easier and more accurate
if (!strEndsWith(status, CR_STR))
THROW_FMT(FormatError, "HTTP response status '%s' should be CR-terminated", strZ(status));
status = strSubN(status, 0, strSize(status) - 1);
// Check status is at least the minimum required length to avoid harder to interpret errors later on
if (strSize(status) < sizeof(HTTP_VERSION) + 4)
THROW_FMT(FormatError, "HTTP response '%s' has invalid length", strZ(strTrim(status)));
// If HTTP/1.0 then the connection will be closed on content eof since connections are not reused by default
if (strBeginsWith(status, HTTP_VERSION_10_STR))
{
this->closeOnContentEof = true;
}
// Else check that the version is the default (1.1)
else if (!strBeginsWith(status, HTTP_VERSION_STR))
THROW_FMT(FormatError, "HTTP version of response '%s' must be " HTTP_VERSION " or " HTTP_VERSION_10, strZ(status));
// Read status code
status = strSub(status, sizeof(HTTP_VERSION));
const int spacePos = strChr(status, ' ');
if (spacePos != 3)
THROW_FMT(FormatError, "response status '%s' must have a space after the status code", strZ(status));
this->pub.code = cvtZSubNToUInt(strZ(status), 0, (size_t)spacePos);
// Read reason phrase. A missing reason phrase will be represented as an empty string.
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pub.reason = strSub(status, (size_t)spacePos + 1);
}
MEM_CONTEXT_OBJ_END();
// Read headers
do
{
// Read the next header
String *const header = strTrim(ioReadLine(httpSessionIoReadP(this->session)));
// If the header is empty then we have reached the end of the headers
if (strSize(header) == 0)
break;
// Split the header and store it
const int colonPos = strChr(header, ':');
if (colonPos < 0)
THROW_FMT(FormatError, "header '%s' missing colon", strZ(strTrim(header)));
const String *const headerKey = strLower(strTrim(strSubN(header, 0, (size_t)colonPos)));
String *const headerValue = strTrim(strSub(header, (size_t)colonPos + 1));
// Read transfer encoding (only chunked is supported)
if (strEq(headerKey, HTTP_HEADER_TRANSFER_ENCODING_STR))
{
// Error if transfer encoding is not chunked
if (!strEq(headerValue, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR))
{
THROW_FMT(
FormatError, "only '%s' is supported for '%s' header", HTTP_VALUE_TRANSFER_ENCODING_CHUNKED,
HTTP_HEADER_TRANSFER_ENCODING);
}
this->contentChunked = true;
}
// Read content size
if (strEq(headerKey, HTTP_HEADER_CONTENT_LENGTH_STR))
{
this->contentSize = cvtZToUInt64(strZ(headerValue));
this->contentRemaining = this->contentSize;
}
// If the server notified of a closed connection then close the client connection after reading content. This
// prevents doing a retry on the next request when using the closed connection.
if (strEq(headerKey, HTTP_HEADER_CONNECTION_STR) && strEq(strLower(headerValue), HTTP_VALUE_CONNECTION_CLOSE_STR))
this->closeOnContentEof = true;
// Add after header checks in case the value was modified
httpHeaderAdd(this->pub.header, headerKey, headerValue);
}
while (1);
// Error if transfer encoding and content length are both set
if (this->contentChunked && this->contentSize > 0)
{
THROW_FMT(
FormatError, "'%s' and '%s' headers are both set", HTTP_HEADER_TRANSFER_ENCODING,
HTTP_HEADER_CONTENT_LENGTH);
}
// Was content returned in the response? HEAD will report content but not actually return any.
this->contentExists =
(this->contentChunked || this->contentSize > 0 || this->closeOnContentEof) && !strEq(verb, HTTP_VERB_HEAD_STR);
this->contentEof = !this->contentExists;
// Create an io object, even if there is no content. This makes the logic for readers easier -- they can just check eof
// rather than also checking if the io object exists.
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pub.contentRead = ioReadNewP(this, .eof = httpResponseEof, .read = httpResponseRead);
ioReadOpen(httpResponseIoRead(this));
}
MEM_CONTEXT_OBJ_END();
// If there is no content then we are done with the client
if (!this->contentExists)
{
httpResponseDone(this);
}
// Else cache content when requested or on error
else if (contentCache || !httpResponseCodeOk(this))
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
httpResponseContent(this);
}
MEM_CONTEXT_OBJ_END();
}
}
MEM_CONTEXT_TEMP_END();
this->pub.contentRead = ioReadNewP(this, .eof = httpResponseEof, .read = httpResponseRead);
ioReadOpen(httpResponseIoRead(this));
}
MEM_CONTEXT_OBJ_END();
// If there is no content then we are done with the client
if (!this->contentExists)
{
httpResponseDone(this);
}
// Else cache content when requested or on error
else if (contentCache || !httpResponseCodeOk(this))
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
httpResponseContent(this);
}
MEM_CONTEXT_OBJ_END();
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(HTTP_RESPONSE, this);
}
@ -394,6 +451,120 @@ httpResponseContent(HttpResponse *const this)
FUNCTION_TEST_RETURN(BUFFER, this->content);
}
/**********************************************************************************************************************************/
FN_EXTERN HttpResponseMulti *
httpResponseMultiNew(const Buffer *const content, const String *const contentType)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BUFFER, content);
FUNCTION_LOG_PARAM(STRING, contentType);
FUNCTION_LOG_END();
ASSERT(content != NULL);
OBJ_NEW_BEGIN(HttpResponseMulti, .childQty = MEM_CONTEXT_QTY_MAX)
{
*this = (HttpResponseMulti)
{
.content = content,
};
MEM_CONTEXT_TEMP_BEGIN()
{
// Extract boundary from content type header
if (contentType == NULL || !strBeginsWith(contentType, STRDEF(HTTP_HEADER_CONTENT_TYPE_MULTIPART)))
THROW(FormatError, "expected multipart content type");
const String *boundary = strSub(contentType, sizeof(HTTP_HEADER_CONTENT_TYPE_MULTIPART) - 1);
if (strZ(boundary)[0] == '"')
boundary = strSubN(boundary, 1, strSize(boundary) - 2);
const String *const boundaryNext = strNewFmt(HTTP_MULTIPART_BOUNDARY_PRE "%s", strZ(boundary));
MEM_CONTEXT_PRIOR_BEGIN()
{
this->boundary = bufNewC(strZ(boundaryNext), strSize(boundaryNext));
}
MEM_CONTEXT_PRIOR_END();
// Find first boundary
const String *const boundaryFirst = strNewFmt("--%s" HTTP_MULTIPART_BOUNDARY_POST, strZ(boundary));
ASSERT(strSize(boundaryFirst) == strSize(boundaryNext));
this->boundaryLast = bufFindP(this->content, BUFSTR(boundaryFirst));
if (this->boundaryLast == NULL)
THROW_FMT(FormatError, "multipart boundary '%s' not found", strZ(boundary));
}
MEM_CONTEXT_TEMP_END();
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(HTTP_RESPONSE_MULTI, this);
}
/**********************************************************************************************************************************/
FN_EXTERN HttpResponse *
httpResponseMultiNext(HttpResponseMulti *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(HTTP_RESPONSE_MULTI, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
HttpResponse *result = NULL;
// Find next boundary
this->boundaryLast += bufSize(this->boundary);
const unsigned char *const boundaryNext = bufFindP(this->content, this->boundary, .begin = this->boundaryLast);
if (boundaryNext != NULL)
{
result = httpResponseNewInternal();
MEM_CONTEXT_TEMP_BEGIN()
{
// Create a buffer for the response
Buffer *const response = bufNewC(this->boundaryLast, (size_t)(boundaryNext - this->boundaryLast));
IoRead *const responseIo = ioBufferReadNewOpen(response);
// Read multipart headers
httpResponseHeaderRead(result, responseIo);
CHECK(
FormatError,
strEq(httpHeaderGet(httpResponseHeader(result), HTTP_HEADER_CONTENT_TYPE_STR), HTTP_HEADER_CONTENT_TYPE_HTTP_STR),
"expected '" HTTP_HEADER_CONTENT_TYPE ":" HTTP_HEADER_CONTENT_TYPE_HTTP "'");
// Read status
httpResponseStatusRead(result, responseIo);
// Read headers
httpResponseHeaderRead(result, responseIo);
// Read content
CHECK(FormatError, !result->contentChunked, "chunked encoding not supported in multipart");
MEM_CONTEXT_OBJ_BEGIN(result)
{
result->content = bufNew((size_t)result->contentSize);
}
MEM_CONTEXT_OBJ_END();
IoWrite *const write = ioBufferWriteNewOpen(result->content);
ioCopyP(responseIo, write);
ioWriteClose(write);
// Set last boundary
this->boundaryLast = boundaryNext + 2;
}
MEM_CONTEXT_TEMP_END();
}
FUNCTION_LOG_RETURN(HTTP_RESPONSE, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
httpResponseToLog(const HttpResponse *const this, StringStatic *const debugLog)
@ -402,9 +573,9 @@ httpResponseToLog(const HttpResponse *const this, StringStatic *const debugLog)
debugLog,
"{code: %u, reason: %s, contentChunked: %s, contentSize: %" PRIu64 ", contentRemaining: %" PRIu64 ", closeOnContentEof: %s"
", contentExists: %s, contentEof: %s, contentCached: %s}",
httpResponseCode(this), strZ(httpResponseReason(this)), cvtBoolToConstZ(this->contentChunked), this->contentSize,
this->contentRemaining, cvtBoolToConstZ(this->closeOnContentEof), cvtBoolToConstZ(this->contentExists),
cvtBoolToConstZ(this->contentEof), cvtBoolToConstZ(this->content != NULL));
httpResponseCode(this), httpResponseReason(this) == NULL ? NULL_Z : strZ(httpResponseReason(this)),
cvtBoolToConstZ(this->contentChunked), this->contentSize, this->contentRemaining, cvtBoolToConstZ(this->closeOnContentEof),
cvtBoolToConstZ(this->contentExists), cvtBoolToConstZ(this->contentEof), cvtBoolToConstZ(this->content != NULL));
strStcCat(debugLog, ", header: "),
httpHeaderToLog(httpResponseHeader(this), debugLog);

40
src/common/io/http/response.h
View File

@ -11,6 +11,7 @@ cached content, etc. will still be available for the lifetime of the object.
Object type
***********************************************************************************************************************************/
typedef struct HttpResponse HttpResponse;
typedef struct HttpResponseMulti HttpResponseMulti;
#include "common/io/http/header.h"
#include "common/io/http/session.h"
@ -24,13 +25,19 @@ HTTP Response Constants
#define HTTP_RESPONSE_CODE_FORBIDDEN 403
#define HTTP_RESPONSE_CODE_NOT_FOUND 404
// 2xx indicates success
#define HTTP_RESPONSE_CODE_CLASS_OK 2
// 5xx errors that should always be retried
#define HTTP_RESPONSE_CODE_CLASS_RETRY 5
/***********************************************************************************************************************************
Constructors
Response Constructors
***********************************************************************************************************************************/
FN_EXTERN HttpResponse *httpResponseNew(HttpSession *session, const String *verb, bool contentCache);
/***********************************************************************************************************************************
Getters/Setters
Response Getters/Setters
***********************************************************************************************************************************/
typedef struct HttpResponsePub
{
@ -70,13 +77,20 @@ httpResponseReason(const HttpResponse *const this)
}
/***********************************************************************************************************************************
Functions
Response Functions
***********************************************************************************************************************************/
// Is this response code OK, i.e. 2XX?
FN_INLINE_ALWAYS bool
httpResponseCodeOk(const HttpResponse *const this)
{
return httpResponseCode(this) / 100 == 2;
return httpResponseCode(this) / 100 == HTTP_RESPONSE_CODE_CLASS_OK;
}
// Should the request be retried?
FN_INLINE_ALWAYS bool
httpResponseCodeRetry(const HttpResponse *const this)
{
return httpResponseCode(this) / 100 == HTTP_RESPONSE_CODE_CLASS_RETRY;
}
// Fetch all response content. Content will be cached so it can be retrieved again without additional cost.
@ -90,7 +104,7 @@ httpResponseMove(HttpResponse *const this, MemContext *const parentNew)
}
/***********************************************************************************************************************************
Destructor
Response Destructor
***********************************************************************************************************************************/
FN_INLINE_ALWAYS void
httpResponseFree(HttpResponse *const this)
@ -98,6 +112,17 @@ httpResponseFree(HttpResponse *const this)
objFree(this);
}
/***********************************************************************************************************************************
Response Multi Constructors
***********************************************************************************************************************************/
FN_EXTERN HttpResponseMulti *httpResponseMultiNew(const Buffer *content, const String *contentType);
/***********************************************************************************************************************************
Response Multi Functions
***********************************************************************************************************************************/
// Get next response
FN_EXTERN HttpResponse *httpResponseMultiNext(HttpResponseMulti *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
@ -108,4 +133,9 @@ FN_EXTERN void httpResponseToLog(const HttpResponse *this, StringStatic *debugLo
#define FUNCTION_LOG_HTTP_RESPONSE_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_OBJECT_FORMAT(value, httpResponseToLog, buffer, bufferSize)
#define FUNCTION_LOG_HTTP_RESPONSE_MULTI_TYPE \
HttpResponseMulti *
#define FUNCTION_LOG_HTTP_RESPONSE_MULTI_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "HttpResponseMulti", buffer, bufferSize)
#endif

36
src/common/type/buffer.c
View File

@ -202,6 +202,42 @@ bufEq(const Buffer *const this, const Buffer *const compare)
FUNCTION_TEST_RETURN(BOOL, false);
}
/**********************************************************************************************************************************/
FN_EXTERN const unsigned char *
bufFind(const Buffer *const this, const Buffer *const find, const BufFindParam param)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BUFFER, this);
FUNCTION_TEST_PARAM(BUFFER, find);
FUNCTION_TEST_PARAM_P(VOID, param.begin);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(find != NULL);
ASSERT(param.begin == NULL || (param.begin >= bufPtrConst(this) && param.begin - bufPtrConst(this) <= (off_t)bufUsed(this)));
const void *result = NULL;
if (bufUsed(this) >= bufUsed(find))
{
const unsigned char *haystack = param.begin != NULL ? param.begin : bufPtrConst(this);
unsigned int findIdx = (unsigned int)(haystack - bufPtrConst(this));
for (; findIdx <= bufUsed(this) - bufUsed(find); findIdx++)
{
if (memcmp(haystack, bufPtrConst(find), bufSize(find)) == 0)
{
result = haystack;
break;
}
haystack++;
}
}
FUNCTION_TEST_RETURN_CONST_P(UCHARDATA, result);
}
/**********************************************************************************************************************************/
FN_EXTERN Buffer *
bufResize(Buffer *const this, const size_t size)

12
src/common/type/buffer.h
View File

@ -125,6 +125,18 @@ FN_EXTERN Buffer *bufCatSub(Buffer *this, const Buffer *cat, size_t catOffset, s
// Are two buffers equal?
FN_EXTERN bool bufEq(const Buffer *this, const Buffer *compare);
// Find a buffer in another buffer
typedef struct BufFindParam
{
VAR_PARAM_HEADER;
const unsigned char *begin; // Begin find from this address
} BufFindParam;
#define bufFindP(this, find, ...) \
bufFind(this, find, (BufFindParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN const unsigned char *bufFind(const Buffer *this, const Buffer *find, BufFindParam param);
// Move to a new parent mem context
FN_INLINE_ALWAYS Buffer *
bufMove(Buffer *const this, MemContext *const parentNew)

228
src/storage/gcs/storage.c
View File

@ -19,12 +19,18 @@ GCS Storage
#include "common/io/tls/client.h"
#include "common/log.h"
#include "common/regExp.h"
#include "common/stat.h"
#include "common/type/json.h"
#include "common/type/object.h"
#include "storage/gcs/read.h"
#include "storage/gcs/write.h"
#include "storage/posix/storage.h"
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_GCS_DELETE_MAX 100
/***********************************************************************************************************************************
HTTP headers
***********************************************************************************************************************************/
@ -73,6 +79,14 @@ VARIANT_STRDEF_STATIC(GCS_JSON_UPDATED_VAR, GCS_JSON_UPD
STRING_STATIC(GCS_FIELD_LIST_MIN_STR, GCS_FIELD_LIST ")");
STRING_STATIC(GCS_FIELD_LIST_MAX_STR, GCS_FIELD_LIST "," GCS_JSON_SIZE "," GCS_JSON_UPDATED ")");
/***********************************************************************************************************************************
Statistics constants
***********************************************************************************************************************************/
STRING_STATIC(GCS_STAT_REMOVE_STR, "gcs.rm");
STRING_STATIC(GCS_STAT_REMOVE_BATCH_STR, "gcs.rm.batch");
STRING_STATIC(GCS_STAT_REMOVE_BATCH_PART_STR, "gcs.rm.batch.part");
STRING_STATIC(GCS_STAT_REMOVE_BATCH_RETRY_STR, "gcs.rm.batch.retry");
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -87,6 +101,7 @@ struct StorageGcs
const String *bucket; // Bucket to store data in
const String *endpoint; // Endpoint
size_t chunkSize; // Block size for resumable upload
unsigned int deleteMax; // Maximum objects that can be deleted in one request
const Buffer *tag; // Tags to be applied to objects
StorageGcsKeyType keyType; // Auth key type
@ -383,6 +398,36 @@ storageGcsAuth(StorageGcs *this, HttpHeader *httpHeader)
/***********************************************************************************************************************************
Process Gcs request
***********************************************************************************************************************************/
// Helper to generate request path
static String *
storageGcsRequestPath(StorageGcs *const this, const String *const object, const bool bucket, const bool upload)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_GCS, this);
FUNCTION_TEST_PARAM(STRING, object);
FUNCTION_TEST_PARAM(BOOL, bucket);
FUNCTION_TEST_PARAM(BOOL, upload);
FUNCTION_TEST_END();
ASSERT(this != NULL);
String *const result = strNew();
MEM_CONTEXT_TEMP_BEGIN()
{
strCatFmt(result, "%s/storage/v1/b", upload ? "/upload" : "");
if (bucket)
strCatFmt(result, "/%s/o", strZ(this->bucket));
if (object != NULL)
strCatFmt(result, "/%s", strZ(httpUriEncode(strSub(object, 1), false)));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN(STRING, result);
}
FN_EXTERN HttpRequest *
storageGcsRequestAsync(StorageGcs *this, const String *verb, StorageGcsRequestAsyncParam param)
{
@ -393,10 +438,12 @@ storageGcsRequestAsync(StorageGcs *this, const String *verb, StorageGcsRequestAs
FUNCTION_LOG_PARAM(BOOL, param.upload);
FUNCTION_LOG_PARAM(BOOL, param.noAuth);
FUNCTION_LOG_PARAM(BOOL, param.tag);
FUNCTION_LOG_PARAM(STRING, param.path);
FUNCTION_LOG_PARAM(STRING, param.object);
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_PARAM(LIST, param.contentList);
FUNCTION_LOG_END();
ASSERT(this != NULL);
@ -408,13 +455,8 @@ storageGcsRequestAsync(StorageGcs *this, const String *verb, StorageGcsRequestAs
MEM_CONTEXT_TEMP_BEGIN()
{
// Generate path
String *path = strCatFmt(strNew(), "%s/storage/v1/b", param.upload ? "/upload" : "");
if (!param.noBucket)
strCatFmt(path, "/%s/o", strZ(this->bucket));
if (param.object != NULL)
strCatFmt(path, "/%s", strZ(httpUriEncode(strSub(param.object, 1), false)));
const String *const path =
param.path != NULL ? param.path : storageGcsRequestPath(this, param.object, !param.noBucket, param.upload);
// Create header list
HttpHeader *requestHeader =
@ -433,10 +475,34 @@ storageGcsRequestAsync(StorageGcs *this, const String *verb, StorageGcsRequestAs
// Set host
httpHeaderPut(requestHeader, HTTP_HEADER_HOST_STR, this->endpoint);
// Set content or construct multipart content
const Buffer *content = param.content;
if (param.contentList != NULL)
{
ASSERT(param.content == NULL);
HttpRequestMulti *const requestMulti = httpRequestMultiNew();
for (unsigned int contentIdx = 0; contentIdx < lstSize(param.contentList); contentIdx++)
{
const StorageGcsRequestPart *const requestPart = lstGet(param.contentList, contentIdx);
HttpHeader *const partHeader = httpHeaderNew(this->headerRedactList);
httpHeaderAdd(partHeader, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
httpRequestMultiAddP(
requestMulti, strNewFmt("%u", contentIdx), requestPart->verb,
storageGcsRequestPath(this, requestPart->object, true, false), .header = partHeader);
}
httpRequestMultiHeaderAdd(requestMulti, requestHeader);
content = httpRequestMultiContent(requestMulti);
}
// Set content length
httpHeaderPut(
requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR,
param.content == NULL || bufEmpty(param.content) ? ZERO_STR : strNewFmt("%zu", bufUsed(param.content)));
content == NULL || bufEmpty(content) ? ZERO_STR : strNewFmt("%zu", bufUsed(content)));
// Make a copy of the query so it can be modified
HttpQuery *query = httpQueryDupP(param.query, .redactList = this->queryRedactList);
@ -449,7 +515,7 @@ storageGcsRequestAsync(StorageGcs *this, const String *verb, StorageGcsRequestAs
MEM_CONTEXT_PRIOR_BEGIN()
{
result = httpRequestNewP(
this->httpClient, verb, path, .query = query, .header = requestHeader, .content = param.content);
this->httpClient, verb, path, .query = query, .header = requestHeader, .content = content);
}
MEM_CONTEXT_END();
}
@ -501,6 +567,7 @@ storageGcsRequest(StorageGcs *const this, const String *const verb, const Storag
FUNCTION_LOG_PARAM(BOOL, param.upload);
FUNCTION_LOG_PARAM(BOOL, param.noAuth);
FUNCTION_LOG_PARAM(BOOL, param.tag);
FUNCTION_LOG_PARAM(STRING, param.path);
FUNCTION_LOG_PARAM(STRING, param.object);
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
@ -512,7 +579,8 @@ storageGcsRequest(StorageGcs *const this, const String *const verb, const Storag
HttpRequest *const request = storageGcsRequestAsyncP(
this, verb, .noBucket = param.noBucket, .upload = param.upload, .noAuth = param.noAuth, .tag = param.tag,
.object = param.object, .header = param.header, .query = param.query, .content = param.content);
.path = param.path, .object = param.object, .header = param.header, .query = param.query, .content = param.content,
.contentList = param.contentList);
HttpResponse *const result = storageGcsResponseP(
request, .allowMissing = param.allowMissing, .allowIncomplete = param.allowIncomplete, .contentIo = param.contentIo);
@ -853,14 +921,100 @@ storageGcsNewWrite(THIS_VOID, const String *file, StorageInterfaceNewWriteParam
}
/**********************************************************************************************************************************/
#define GCS_HEADER_CONTENTID_RESPONSE "response-"
typedef struct StorageGcsPathRemoveData
{
StorageGcs *this; // Storage Object
MemContext *memContext; // Mem context to create requests in
HttpRequest *request; // Async remove request
List *requestContentList; // Content list for async request
List *contentList; // Content list currently being built
const String *path; // Root path of remove
} StorageGcsPathRemoveData;
static void
storageGcsPathRemoveInternal(StorageGcsPathRemoveData *const data)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_END();
ASSERT(data != NULL);
ASSERT(data->this != NULL);
// Get response for async request
if (data->request != NULL)
{
MEM_CONTEXT_TEMP_BEGIN()
{
HttpResponse *const response = storageGcsResponseP(data->request);
HttpResponseMulti *const responseMulti = httpResponseMultiNew(
httpResponseContent(response), httpHeaderGet(httpResponseHeader(response), HTTP_HEADER_CONTENT_TYPE_STR));
// Loop through all response parts
HttpResponse *responsePart = httpResponseMultiNext(responseMulti);
CHECK(FormatError, responsePart != NULL, "at least one response part is required");
do
{
// If not OK and not missing then retry
if (!httpResponseCodeOk(responsePart) && httpResponseCode(responsePart) != HTTP_RESPONSE_CODE_NOT_FOUND)
{
// Extract and check content-id header
const String *const contentId = httpHeaderGet(httpResponseHeader(responsePart), HTTP_HEADER_CONTENT_ID_STR);
CHECK(FormatError, contentId != NULL, HTTP_HEADER_CONTENT_ID " header is not present");
CHECK_FMT(
FormatError,
strBeginsWithZ(contentId, GCS_HEADER_CONTENTID_RESPONSE),
HTTP_HEADER_CONTENT_ID " header '%s' must begin with '" GCS_HEADER_CONTENTID_RESPONSE "'", strZ(contentId));
// Use content-id to get content
const unsigned int contentIdx = cvtZToUInt(strZ(strSub(contentId, sizeof(GCS_HEADER_CONTENTID_RESPONSE) - 1)));
const StorageGcsRequestPart *const content = lstGet(data->requestContentList, contentIdx);
// Retry remove
statInc(GCS_STAT_REMOVE_BATCH_RETRY_STR);
storageGcsRequestP(data->this, content->verb, .object = content->object, .allowMissing = true);
}
else
statInc(GCS_STAT_REMOVE_BATCH_PART_STR);
httpResponseFree(responsePart);
responsePart = httpResponseMultiNext(responseMulti);
}
while (responsePart != NULL);
}
MEM_CONTEXT_TEMP_END();
// Free request
httpRequestFree(data->request);
data->request = NULL;
// Free content list
lstFree(data->requestContentList);
}
// Send new async request if there is more to remove
if (data->contentList != NULL)
{
statInc(GCS_STAT_REMOVE_BATCH_STR);
MEM_CONTEXT_BEGIN(data->memContext)
{
data->request = storageGcsRequestAsyncP(
data->this, HTTP_VERB_POST_STR, .path = STRDEF("/batch/storage/v1"), .contentList = data->contentList);
}
MEM_CONTEXT_END();
// Store the content list for use in error handling
data->requestContentList = data->contentList;
data->contentList = NULL;
}
FUNCTION_TEST_RETURN_VOID();
}
static void
storageGcsPathRemoveCallback(void *const callbackData, const StorageInfo *const info)
{
@ -872,25 +1026,34 @@ storageGcsPathRemoveCallback(void *const callbackData, const StorageInfo *const
ASSERT(callbackData != NULL);
ASSERT(info != NULL);
StorageGcsPathRemoveData *const data = callbackData;
// Get response from prior async request
if (data->request != NULL)
{
httpResponseFree(storageGcsResponseP(data->request, .allowMissing = true));
httpRequestFree(data->request);
data->request = NULL;
}
// Only delete files since paths don't really exist
if (info->type == storageTypeFile)
{
MEM_CONTEXT_BEGIN(data->memContext)
StorageGcsPathRemoveData *const data = callbackData;
if (data->contentList == NULL)
{
data->request = storageGcsRequestAsyncP(
data->this, HTTP_VERB_DELETE_STR, .object = strNewFmt("%s/%s", strZ(data->path), strZ(info->name)));
MEM_CONTEXT_BEGIN(data->memContext)
{
data->contentList = lstNewP(sizeof(StorageGcsRequestPart));
}
MEM_CONTEXT_END();
}
MEM_CONTEXT_END();
MEM_CONTEXT_OBJ_BEGIN(data->contentList)
{
const StorageGcsRequestPart content =
{
.verb = HTTP_VERB_DELETE_STR,
.object = strNewFmt("%s/%s", strZ(data->path), strZ(info->name)),
};
lstAdd(data->contentList, &content);
}
MEM_CONTEXT_OBJ_END();
if (lstSize(data->contentList) == data->this->deleteMax)
storageGcsPathRemoveInternal(data);
}
FUNCTION_TEST_RETURN_VOID();
@ -920,11 +1083,18 @@ storageGcsPathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfa
.path = strEq(path, FSLASH_STR) ? EMPTY_STR : path,
};
storageGcsListInternal(this, path, storageInfoLevelType, NULL, true, storageGcsPathRemoveCallback, &data);
MEM_CONTEXT_TEMP_BEGIN()
{
storageGcsListInternal(this, path, storageInfoLevelType, NULL, true, storageGcsPathRemoveCallback, &data);
// Check response on last async request
if (data.request != NULL)
storageGcsResponseP(data.request, .allowMissing = true);
// Call if there is more to be removed
if (data.contentList != NULL)
storageGcsPathRemoveInternal(&data);
// Check response on last async request
storageGcsPathRemoveInternal(&data);
}
MEM_CONTEXT_TEMP_END();
}
MEM_CONTEXT_TEMP_END();
@ -947,6 +1117,7 @@ storageGcsRemove(THIS_VOID, const String *const file, const StorageInterfaceRemo
ASSERT(file != NULL);
ASSERT(!param.errorOnMissing);
statInc(GCS_STAT_REMOVE_STR);
httpResponseFree(storageGcsRequestP(this, HTTP_VERB_DELETE_STR, .object = file, .allowMissing = true));
FUNCTION_LOG_RETURN_VOID();
@ -1000,6 +1171,7 @@ storageGcsNew(
.bucket = strDup(bucket),
.keyType = keyType,
.chunkSize = chunkSize,
.deleteMax = STORAGE_GCS_DELETE_MAX,
};
// Create tag JSON buffer

13
src/storage/gcs/storage.intern.h
View File

@ -40,6 +40,15 @@ VARIANT_DECLARE(GCS_JSON_NAME_VAR);
#define GCS_JSON_SIZE "size"
VARIANT_DECLARE(GCS_JSON_SIZE_VAR);
/***********************************************************************************************************************************
Multi-Part request data
***********************************************************************************************************************************/
typedef struct StorageGcsRequestPart
{
const String *object; // Object to include in URI
const String *verb; // Verb (GET, PUT, etc)
} StorageGcsRequestPart;
/***********************************************************************************************************************************
Perform a GCS Request
***********************************************************************************************************************************/
@ -51,10 +60,12 @@ typedef struct StorageGcsRequestAsyncParam
bool upload; // Is an object upload?
bool noAuth; // Exclude authentication header?
bool tag; // Add tags when available?
const String *path; // URI path (this overrides object)
const String *object; // Object to include in URI
const HttpHeader *header; // Request headers
const HttpQuery *query; // Query parameters
const Buffer *content; // Request content
const List *contentList; // Request content part list
} StorageGcsRequestAsyncParam;
#define storageGcsRequestAsyncP(this, verb, ...) \
@ -83,10 +94,12 @@ typedef struct StorageGcsRequestParam
bool upload; // Is an object upload?
bool noAuth; // Exclude authentication header?
bool tag; // Add tags when available?
const String *path; // URI path (this overrides object)
const String *object; // Object to include in URI
const HttpHeader *header; // Request headers
const HttpQuery *query; // Query parameters
const Buffer *content; // Request content
const List *contentList; // Request content part list
bool allowMissing; // Allow missing files (caller can check response code)
bool allowIncomplete; // Allow incomplete resume (used for resumable upload)
bool contentIo; // Is IoRead interface required to read content?

2
test/define.yaml
View File

@ -393,7 +393,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: io-http
total: 6
total: 7
coverage:
- common/io/http/client

124
test/src/module/common/ioHttpTest.c
View File

@ -309,6 +309,19 @@ testRun(void)
TEST_RESULT_VOID(httpUrlFree(url), "free");
}
// *****************************************************************************************************************************
if (testBegin("HttpResponseMulti"))
{
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("errors");
TEST_ERROR(httpResponseMultiNew(BUFSTRDEF(""), NULL), FormatError, "expected multipart content type");
TEST_ERROR(httpResponseMultiNew(BUFSTRDEF(""), STRDEF("bogus")), FormatError, "expected multipart content type");
TEST_ERROR(
strNewBuf(httpResponseMultiNew(BUFSTRDEF("--YYY"), STRDEF("multipart/mixed; boundary=\"XXX\""))->boundary),
FormatError, "multipart boundary 'XXX' not found");
}
// *****************************************************************************************************************************
if (testBegin("HttpClient"))
{
@ -842,9 +855,118 @@ testRun(void)
TEST_RESULT_STR_Z(strNewBuf(buffer), "", "check response");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("close connection and end server process");
TEST_TITLE("request with multipart request and response");
ioBufferSizeSet(512);
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(
http,
"GET / HTTP/1.1\r\n" TEST_USER_AGENT
"content-type:multipart/mixed; boundary=QKX4EYg4LARJ\r\n"
"hdr1:1\r\n\r\n"
"\r\n--QKX4EYg4LARJ\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:0\r\n\r\n"
"GET / HTTP/1.1\r\n\r\n"
"\r\n--QKX4EYg4LARJ\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:1\r\n\r\n"
"POST /ack HTTP/1.1\r\n"
"content-length:3\r\n\r\n"
HTTP_MULTIPART_BOUNDARY_INIT
"\r\n--QKX4EYg4LARJ--\r\n");
hrnServerScriptReplyZ(
http,
"HTTP/1.1 200 OK\r\nConnection:ClosE\r\ncontent-type:multipart/mixed; boundary=XXX\r\n\r\n"
"PREAMBLE JUNK\r\n"
"--XXX\r\n"
"content-type:application/http\r\n"
"content-id:0\r\n\r\n"
"HTTP/1.1 200 OK\r\n\r\n"
"\r\n--XXX\r\n"
"content-type:application/http\r\n"
"content-id:1\r\n"
"\r\n"
"HTTP/1.1 200 OK\r\n"
"content-length:3\r\n"
"\r\n"
"123"
"\r\n--XXX\n"
"EPILOGUE JUNK");
hrnServerScriptClose(http);
HttpRequestMulti *requestMulti = httpRequestMultiNew();
httpRequestMultiAddP(requestMulti, STRDEF("0"), HTTP_VERB_GET_STR, STRDEF("/"), .header = httpHeaderNew(NULL));
httpRequestMultiAddP(
requestMulti, STRDEF("1"), HTTP_VERB_POST_STR, STRDEF("/ack"),
.header = httpHeaderAdd(httpHeaderNew(NULL), HTTP_HEADER_CONTENT_LENGTH_STR, STRDEF("3")),
.content = BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_INIT));
HttpHeader *headerRequest = httpHeaderNew(NULL);
httpHeaderAdd(headerRequest, STRDEF("hdr1"), STRDEF("1"));
httpRequestMultiHeaderAdd(requestMulti, headerRequest);
TEST_ASSIGN(
response,
httpRequestResponse(
httpRequestNewP(
client, STRDEF("GET"), STRDEF("/"), .header = headerRequest,
.content = httpRequestMultiContent(requestMulti)),
false),
"request");
TEST_RESULT_VOID(
FUNCTION_LOG_OBJECT_FORMAT(httpResponseHeader(response), httpHeaderToLog, logBuf, sizeof(logBuf)),
"httpHeaderToLog");
TEST_RESULT_Z(
logBuf, "{connection: 'close', content-type: 'multipart/mixed; boundary=XXX'}", "check response headers");
HttpResponseMulti *responseMulti = NULL;
TEST_ASSIGN(
responseMulti,
httpResponseMultiNew(
httpResponseContent(response), httpHeaderGet(httpResponseHeader(response), HTTP_HEADER_CONTENT_TYPE_STR)),
"response multi");
HttpResponse *responsePart = NULL;
TEST_ASSIGN(responsePart, httpResponseMultiNext(responseMulti), "response part");
TEST_RESULT_UINT(httpResponseCode(responsePart), 200, "response code");
TEST_RESULT_STR_Z(strNewBuf(httpResponseContent(responsePart)), "", "response content");
TEST_ASSIGN(responsePart, httpResponseMultiNext(responseMulti), "response part");
TEST_RESULT_STR_Z(strNewBuf(httpResponseContent(responsePart)), "123", "response content");
TEST_RESULT_PTR(httpResponseMultiNext(responseMulti), NULL, "no more responses");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on full boundary in header or content");
requestMulti = httpRequestMultiNew();
TEST_ERROR(
httpRequestMultiAddP(
requestMulti, STRDEF("0"), HTTP_VERB_GET_STR, STRDEF("/"),
.header = httpHeaderAdd(
httpHeaderNew(NULL), HTTP_HEADER_CONTENT_LENGTH_STR,
STRDEF(HTTP_MULTIPART_BOUNDARY_INIT HTTP_MULTIPART_BOUNDARY_EXTRA))),
AssertError, "unable to construct unique boundary");
requestMulti = httpRequestMultiNew();
TEST_ERROR(
httpRequestMultiAddP(
requestMulti, STRDEF("0"), HTTP_VERB_GET_STR, STRDEF("/"), .header = httpHeaderNew(NULL),
.content = BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_INIT HTTP_MULTIPART_BOUNDARY_EXTRA)),
AssertError, "unable to construct unique boundary");
TEST_RESULT_BOOL(
bufEq(requestMulti->boundaryRaw, BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_INIT HTTP_MULTIPART_BOUNDARY_EXTRA)),
true, "max boundary");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("end server process");
hrnServerScriptEnd(http);
}
HRN_FORK_PARENT_END();

11
test/src/module/common/typeBufferTest.c
View File

@ -137,11 +137,20 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("bufDup() and bufEq()"))
if (testBegin("bufDup(), bufEq(), and bufFind()"))
{
TEST_RESULT_BOOL(bufEq(BUFSTRDEF("123"), bufDup(BUFSTRDEF("1234"))), false, "buffer sizes not equal");
TEST_RESULT_BOOL(bufEq(BUFSTR(STRDEF("321")), BUFSTRDEF("123")), false, "buffer sizes equal");
TEST_RESULT_BOOL(bufEq(bufDup(BUFSTRZ("123")), BUF("123", 3)), true, "buffers equal");
const Buffer *haystack = BUFSTRDEF("findsomethinginhere");
TEST_RESULT_PTR(bufFindP(haystack, BUFSTRDEF("xxx")), NULL, "not found");
TEST_RESULT_PTR(bufFindP(haystack, BUFSTRDEF("find")), bufPtrConst(haystack), "found first");
TEST_RESULT_PTR(bufFindP(haystack, BUFSTRDEF("here")), bufPtrConst(haystack) + 15, "found last");
TEST_RESULT_PTR(bufFindP(haystack, BUFSTRDEF("thing")), bufPtrConst(haystack) + 8, "found middle");
TEST_RESULT_PTR(bufFindP(haystack, BUFSTRDEF("find"), .begin = bufPtrConst(haystack) + 1), NULL, "skipped not found");
TEST_RESULT_PTR(bufFindP(haystack, BUFSTRDEF("findsomethinginhere2")), NULL, "needle longer than haystack");
}
// *****************************************************************************************************************************

143
test/src/module/storage/gcsTest.c
View File

@ -72,6 +72,8 @@ typedef struct TestRequestParam
bool noBucket;
bool upload;
bool noAuth;
bool multiPart;
const char *path;
const char *object;
const char *query;
const char *contentRange;
@ -86,8 +88,10 @@ typedef struct TestRequestParam
static void
testRequest(IoWrite *write, const char *verb, TestRequestParam param)
{
String *request = strCatFmt(
strNew(), "%s %s/storage/v1/b%s", verb, param.upload ? "/upload" : "", param.noBucket ? "" : "/bucket/o");
String *const request =
param.path != NULL ?
strCatFmt(strNew(), "%s %s", verb, param.path) :
strCatFmt(strNew(), "%s %s/storage/v1/b%s", verb, param.upload ? "/upload" : "", param.noBucket ? "" : "/bucket/o");
// Add object
if (param.object != NULL)
@ -115,6 +119,13 @@ testRequest(IoWrite *write, const char *verb, TestRequestParam param)
if (param.contentType != NULL)
strCatFmt(request, "content-type:%s\r\n", param.contentType);
// Add multipart content-type
if (param.multiPart)
{
ASSERT(param.contentType == NULL);
strCatZ(request, "content-type:multipart/mixed; boundary=" HTTP_MULTIPART_BOUNDARY_INIT "\r\n");
}
// Add host
strCatFmt(request, "host:%s\r\n", strZ(hrnServerHost()));
@ -139,6 +150,7 @@ typedef struct TestResponseParam
{
VAR_PARAM_HEADER;
unsigned int code;
bool multiPart;
const char *header;
const char *content;
} TestResponseParam;
@ -174,6 +186,10 @@ testResponse(IoWrite *write, TestResponseParam param)
if (param.header != NULL)
strCatFmt(response, "%s\r\n", param.header);
// Add multipart content-type
if (param.multiPart)
strCatZ(response, "content-type:multipart/mixed; boundary=" HTTP_MULTIPART_BOUNDARY_INIT "\r\n");
// Content
if (param.content != NULL)
{
@ -940,14 +956,62 @@ testRun(void)
" },"
" {"
" \"name\": \"path1/xxx.zzz\""
" },"
" {"
" \"name\": \"path2/file2\""
" }"
" ]"
"}");
testRequestP(service, HTTP_VERB_DELETE, .object = "path/to/test1.txt");
testResponseP(service);
testRequestP(
service, HTTP_VERB_POST, .path = "/batch/storage/v1", .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:0\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path%2Fto%2Ftest1.txt HTTP/1.1\r\n"
"content-length:0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:1\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path1%2Fxxx.zzz HTTP/1.1\r\n"
"content-length:0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:2\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path2%2Ffile2 HTTP/1.1\r\n"
"content-length:0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
service, .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:response-0\r\n"
"\r\n"
"HTTP/1.1 404 Missing\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:response-1\r\n"
"\r\n"
"HTTP/1.1 200 OK\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:response-2\r\n"
"\r\n"
"HTTP/1.1 300 Error\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testRequestP(service, HTTP_VERB_DELETE, .object = "path1/xxx.zzz");
testRequestP(service, HTTP_VERB_DELETE, .object = "path2/file2");
testResponseP(service);
TEST_RESULT_VOID(storagePathRemoveP(storage, STRDEF("/"), .recurse = true), "remove");
@ -955,6 +1019,8 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("remove files from path");
((StorageGcs *)storageDriver(storage))->deleteMax = 1;
testRequestP(service, HTTP_VERB_GET, .query = "fields=nextPageToken%2Cprefixes%2Citems%28name%29&prefix=path%2F");
testResponseP(
service,
@ -973,13 +1039,72 @@ testRun(void)
" ]"
"}");
testRequestP(service, HTTP_VERB_DELETE, .object = "path/test1.txt");
testResponseP(service);
testRequestP(
service, HTTP_VERB_POST, .path = "/batch/storage/v1", .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:0\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path%2Ftest1.txt HTTP/1.1\r\n"
"content-length:0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
service, .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:response-0\r\n"
"\r\n"
"HTTP/1.1 404 OK\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testRequestP(
service, HTTP_VERB_POST, .path = "/batch/storage/v1", .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:0\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path%2Fpath1%2Fxxx.zzz HTTP/1.1\r\n"
"content-length:0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
service, .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:response-0\r\n"
"\r\n"
"HTTP/1.1 305 Error\r\n"
"content-length:5\r\n"
"content-type:text\r\n\r\n"
"ERROR"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testRequestP(service, HTTP_VERB_DELETE, .object = "path/path1/xxx.zzz");
testResponseP(service);
testResponseP(service, .code = 300, .content = "ERROR2");
TEST_RESULT_VOID(storagePathRemoveP(storage, STRDEF("/path"), .recurse = true), "remove");
TEST_ERROR_FMT(
storagePathRemoveP(storage, STRDEF("/path"), .recurse = true), ProtocolError,
"HTTP request failed with 300:\n"
"*** Path/Query ***:\n"
"DELETE /storage/v1/b/bucket/o/path%%2Fpath1%%2Fxxx.zzz\n"
"*** Request Headers ***:\n"
"authorization: <redacted>\n"
"content-length: 0\n"
"host: %s\n"
"*** Response Headers ***:\n"
"content-length: 6\n"
"*** Response Content ***:\n"
"ERROR2",
strZ(hrnServerHost()));
((StorageGcs *)storageDriver(storage))->deleteMax = STORAGE_GCS_DELETE_MAX;
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("remove files in empty subpath (nothing to do)");