diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 677aafaa3..a5c087bc2 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -15,6 +15,10 @@ + +

Asynchronous S3 multipart upload.

+ + diff --git a/src/Makefile.in b/src/Makefile.in index 8548e02bf..9a3f37cf0 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -73,11 +73,13 @@ SRCS = \ common/io/filter/size.c \ common/io/handleRead.c \ common/io/handleWrite.c \ - common/io/http/cache.c \ common/io/http/client.c \ common/io/http/common.c \ common/io/http/header.c \ common/io/http/query.c \ + common/io/http/request.c \ + common/io/http/response.c \ + common/io/http/session.c \ common/io/io.c \ common/io/read.c \ common/io/socket/client.c \ diff --git a/src/command/repo/create.c b/src/command/repo/create.c index 68b50f033..31539c8ce 100644 --- a/src/command/repo/create.c +++ b/src/command/repo/create.c @@ -19,10 +19,7 @@ cmdRepoCreate(void) MEM_CONTEXT_TEMP_BEGIN() { if (strEq(storageType(storageRepo()), STORAGE_S3_TYPE_STR)) - { - storageS3Request( - (StorageS3 *)storageDriver(storageRepoWrite()), HTTP_VERB_PUT_STR, FSLASH_STR, NULL, NULL, true, false); - } + storageS3RequestP((StorageS3 *)storageDriver(storageRepoWrite()), HTTP_VERB_PUT_STR, FSLASH_STR); } MEM_CONTEXT_TEMP_END(); diff --git a/src/common/io/http/cache.c b/src/common/io/http/cache.c deleted file mode 100644 index 8472a2f60..000000000 --- a/src/common/io/http/cache.c +++ /dev/null @@ -1,104 +0,0 @@ -/*********************************************************************************************************************************** -HTTP Client Cache -***********************************************************************************************************************************/ -#include "build.auto.h" - -#include "common/debug.h" -#include "common/io/http/cache.h" -#include "common/log.h" -#include "common/type/list.h" -#include "common/type/object.h" - -/*********************************************************************************************************************************** -Object type -***********************************************************************************************************************************/ -struct HttpClientCache -{ - MemContext *memContext; // Mem context - - const String *host; // Client settings - unsigned int port; - TimeMSec timeout; - bool verifyPeer; - const String *caFile; - const String *caPath; - - List *clientList; // List of HTTP clients -}; - -OBJECT_DEFINE_FREE(HTTP_CLIENT_CACHE); - -/**********************************************************************************************************************************/ -HttpClientCache * -httpClientCacheNew( - const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath) -{ - FUNCTION_LOG_BEGIN(logLevelDebug) - FUNCTION_LOG_PARAM(STRING, host); - FUNCTION_LOG_PARAM(UINT, port); - FUNCTION_LOG_PARAM(TIME_MSEC, timeout); - FUNCTION_LOG_PARAM(BOOL, verifyPeer); - FUNCTION_LOG_PARAM(STRING, caFile); - FUNCTION_LOG_PARAM(STRING, caPath); - FUNCTION_LOG_END(); - - ASSERT(host != NULL); - - HttpClientCache *this = NULL; - - MEM_CONTEXT_NEW_BEGIN("HttpClientCache") - { - // Allocate state and set context - this = memNew(sizeof(HttpClientCache)); - - *this = (HttpClientCache) - { - .memContext = MEM_CONTEXT_NEW(), - .host = strDup(host), - .port = port, - .timeout = timeout, - .verifyPeer = verifyPeer, - .caFile = strDup(caFile), - .caPath = strDup(caPath), - .clientList = lstNew(sizeof(HttpClient *)), - }; - } - MEM_CONTEXT_NEW_END(); - - FUNCTION_LOG_RETURN(HTTP_CLIENT_CACHE, this); -} - -/**********************************************************************************************************************************/ -HttpClient * -httpClientCacheGet(HttpClientCache *this) -{ - FUNCTION_LOG_BEGIN(logLevelTrace) - FUNCTION_LOG_PARAM(HTTP_CLIENT_CACHE, this); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - - HttpClient *result = NULL; - - // Search for a client that is not busy - for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++) - { - HttpClient *httpClient = *(HttpClient **)lstGet(this->clientList, clientIdx); - - if (!httpClientBusy(httpClient)) - result = httpClient; - } - - // If none found then create a new one - if (result == NULL) - { - MEM_CONTEXT_BEGIN(this->memContext) - { - result = httpClientNew(this->host, this->port, this->timeout, this->verifyPeer, this->caFile, this->caPath); - lstAdd(this->clientList, &result); - } - MEM_CONTEXT_END(); - } - - FUNCTION_LOG_RETURN(HTTP_CLIENT, result); -} diff --git a/src/common/io/http/client.c b/src/common/io/http/client.c index c452846fb..23db50247 100644 --- a/src/common/io/http/client.c +++ b/src/common/io/http/client.c @@ -5,49 +5,14 @@ HTTP Client #include "common/debug.h" #include "common/io/http/client.h" -#include "common/io/http/common.h" -#include "common/io/io.h" -#include "common/io/read.intern.h" #include "common/io/tls/client.h" #include "common/log.h" #include "common/type/object.h" -#include "common/wait.h" - -/*********************************************************************************************************************************** -Http constants -***********************************************************************************************************************************/ -#define HTTP_VERSION "HTTP/1.1" - STRING_STATIC(HTTP_VERSION_STR, HTTP_VERSION); - -STRING_EXTERN(HTTP_VERB_DELETE_STR, HTTP_VERB_DELETE); -STRING_EXTERN(HTTP_VERB_GET_STR, HTTP_VERB_GET); -STRING_EXTERN(HTTP_VERB_HEAD_STR, HTTP_VERB_HEAD); -STRING_EXTERN(HTTP_VERB_POST_STR, HTTP_VERB_POST); -STRING_EXTERN(HTTP_VERB_PUT_STR, HTTP_VERB_PUT); - -STRING_EXTERN(HTTP_HEADER_AUTHORIZATION_STR, HTTP_HEADER_AUTHORIZATION); -#define HTTP_HEADER_CONNECTION "connection" - STRING_STATIC(HTTP_HEADER_CONNECTION_STR, HTTP_HEADER_CONNECTION); -STRING_EXTERN(HTTP_HEADER_CONTENT_LENGTH_STR, HTTP_HEADER_CONTENT_LENGTH); -STRING_EXTERN(HTTP_HEADER_CONTENT_MD5_STR, HTTP_HEADER_CONTENT_MD5); -STRING_EXTERN(HTTP_HEADER_ETAG_STR, HTTP_HEADER_ETAG); -STRING_EXTERN(HTTP_HEADER_HOST_STR, HTTP_HEADER_HOST); -STRING_EXTERN(HTTP_HEADER_LAST_MODIFIED_STR, HTTP_HEADER_LAST_MODIFIED); -#define HTTP_HEADER_TRANSFER_ENCODING "transfer-encoding" - STRING_STATIC(HTTP_HEADER_TRANSFER_ENCODING_STR, HTTP_HEADER_TRANSFER_ENCODING); - -#define HTTP_VALUE_CONNECTION_CLOSE "close" - STRING_STATIC(HTTP_VALUE_CONNECTION_CLOSE_STR, HTTP_VALUE_CONNECTION_CLOSE); -#define HTTP_VALUE_TRANSFER_ENCODING_CHUNKED "chunked" - STRING_STATIC(HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED); - -// 5xx errors that should always be retried -#define HTTP_RESPONSE_CODE_RETRY_CLASS 5 /*********************************************************************************************************************************** Statistics ***********************************************************************************************************************************/ -static HttpClientStat httpClientStatLocal; +HttpClientStat httpClientStat; /*********************************************************************************************************************************** Object type @@ -56,138 +21,12 @@ struct HttpClient { MemContext *memContext; // Mem context TimeMSec timeout; // Request timeout - TlsClient *tlsClient; // TLS client - TlsSession *tlsSession; // Current TLS session - IoRead *ioRead; // Read io interface - unsigned int responseCode; // Response code (e.g. 200, 404) - String *responseMessage; // Response message e.g. (OK, Not Found) - HttpHeader *responseHeader; // Response headers - - bool contentChunked; // Is the response content chunked? - uint64_t contentSize; // Content size (ignored for chunked) - uint64_t contentRemaining; // Content remaining (per chunk if chunked) - bool closeOnContentEof; // Will server close after content is sent? - bool contentEof; // Has all content been read? + List *sessionReuseList; // List of HTTP sessions that can be reused }; -OBJECT_DEFINE_FREE(HTTP_CLIENT); - -/*********************************************************************************************************************************** -Read content -***********************************************************************************************************************************/ -static size_t -httpClientRead(THIS_VOID, Buffer *buffer, bool block) -{ - THIS(HttpClient); - - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(HTTP_CLIENT, this); - FUNCTION_LOG_PARAM(BUFFER, buffer); - FUNCTION_LOG_PARAM(BOOL, block); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(buffer != NULL); - ASSERT(!bufFull(buffer)); - - // Read if EOF has not been reached - size_t actualBytes = 0; - - if (!this->contentEof) - { - // If close was requested and no content specified then the server may send content up until the eof - if (this->closeOnContentEof && !this->contentChunked && this->contentSize == 0) - { - ioRead(tlsSessionIoRead(this->tlsSession), buffer); - this->contentEof = ioReadEof(tlsSessionIoRead(this->tlsSession)); - } - // Else read using specified encoding or size - else - { - do - { - // If chunked content and no content remaining - if (this->contentChunked && this->contentRemaining == 0) - { - // Read length of next chunk - MEM_CONTEXT_TEMP_BEGIN() - { - this->contentRemaining = cvtZToUInt64Base( - strPtr(strTrim(ioReadLine(tlsSessionIoRead(this->tlsSession)))), 16); - } - MEM_CONTEXT_TEMP_END(); - - // If content remaining is still zero then eof - if (this->contentRemaining == 0) - this->contentEof = true; - } - - // Read if there is content remaining - if (this->contentRemaining > 0) - { - // If the buffer is larger than the content that needs to be read then limit the buffer size so the read won't - // block or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is defined as - // less than 2^32 so content remaining can't be more than that. - if (bufRemains(buffer) > this->contentRemaining) - bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining)); - - actualBytes = bufRemains(buffer); - this->contentRemaining -= ioRead(tlsSessionIoRead(this->tlsSession), buffer); - - // Error if EOF but content read is not complete - if (ioReadEof(tlsSessionIoRead(this->tlsSession))) - THROW(FileReadError, "unexpected EOF reading HTTP content"); - - // Clear limit (this works even if the limit was not set and it is easier than checking) - bufLimitClear(buffer); - } - - // If no content remaining - if (this->contentRemaining == 0) - { - // If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back - // around to check. - if (this->contentChunked) - { - ioReadLine(tlsSessionIoRead(this->tlsSession)); - } - // If total content size was provided then this is eof - else - this->contentEof = true; - } - } - while (!bufFull(buffer) && !this->contentEof); - } - - // If the server notified that it would close the connection after sending content then close the client side - if (this->contentEof && this->closeOnContentEof) - { - tlsSessionFree(this->tlsSession); - this->tlsSession = NULL; - } - } - - FUNCTION_LOG_RETURN(SIZE, (size_t)actualBytes); -} - -/*********************************************************************************************************************************** -Has all content been read? -***********************************************************************************************************************************/ -static bool -httpClientEof(THIS_VOID) -{ - THIS(HttpClient); - - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(HTTP_CLIENT, this); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - - FUNCTION_LOG_RETURN(BOOL, this->contentEof); -} +OBJECT_DEFINE_GET(Timeout, const, HTTP_CLIENT, TimeMSec, timeout); /**********************************************************************************************************************************/ HttpClient * @@ -216,9 +55,10 @@ httpClientNew( .memContext = MEM_CONTEXT_NEW(), .timeout = timeout, .tlsClient = tlsClientNew(sckClientNew(host, port, timeout), timeout, verifyPeer, caFile, caPath), + .sessionReuseList = lstNew(sizeof(HttpSession *)), }; - httpClientStatLocal.object++; + httpClientStat.object++; } MEM_CONTEXT_NEW_END(); @@ -226,273 +66,53 @@ httpClientNew( } /**********************************************************************************************************************************/ -Buffer * -httpClientRequest( - HttpClient *this, const String *verb, const String *uri, const HttpQuery *query, const HttpHeader *requestHeader, - const Buffer *body, bool returnContent) +HttpSession * +httpClientOpen(HttpClient *this) { - FUNCTION_LOG_BEGIN(logLevelDebug) + FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(HTTP_CLIENT, this); - FUNCTION_LOG_PARAM(STRING, verb); - FUNCTION_LOG_PARAM(STRING, uri); - FUNCTION_LOG_PARAM(HTTP_QUERY, query); - FUNCTION_LOG_PARAM(HTTP_HEADER, requestHeader); - FUNCTION_LOG_PARAM(BUFFER, body); - FUNCTION_LOG_PARAM(BOOL, returnContent); FUNCTION_LOG_END(); ASSERT(this != NULL); - ASSERT(verb != NULL); - ASSERT(uri != NULL); - // Buffer for returned content - Buffer *result = NULL; + HttpSession *result = NULL; - MEM_CONTEXT_TEMP_BEGIN() + // Check if there is a resuable session + if (lstSize(this->sessionReuseList) > 0) { - bool retry; - Wait *wait = waitNew(this->timeout); + // Remove session from reusable list + result = *(HttpSession **)lstGet(this->sessionReuseList, 0); + lstRemoveIdx(this->sessionReuseList, 0); - do - { - // Assume there will be no retry - retry = false; - - // Free the read interface - httpClientDone(this); - - // Free response status left over from the last request - httpHeaderFree(this->responseHeader); - this->responseHeader = NULL; - strFree(this->responseMessage); - this->responseMessage = NULL; - - // Reset all content info - this->contentChunked = false; - this->contentSize = 0; - this->contentRemaining = 0; - this->closeOnContentEof = false; - this->contentEof = true; - - TRY_BEGIN() - { - if (this->tlsSession == NULL) - { - MEM_CONTEXT_BEGIN(this->memContext) - { - this->tlsSession = tlsClientOpen(this->tlsClient); - httpClientStatLocal.session++; - } - MEM_CONTEXT_END(); - } - - // Write the request - String *queryStr = httpQueryRender(query); - - ioWriteStrLine( - tlsSessionIoWrite(this->tlsSession), - strNewFmt( - "%s %s%s%s " HTTP_VERSION "\r", strPtr(verb), strPtr(httpUriEncode(uri, true)), queryStr == NULL ? "" : "?", - queryStr == NULL ? "" : strPtr(queryStr))); - - // Write headers - if (requestHeader != NULL) - { - const StringList *headerList = httpHeaderList(requestHeader); - - for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++) - { - const String *headerKey = strLstGet(headerList, headerIdx); - ioWriteStrLine( - tlsSessionIoWrite(this->tlsSession), - strNewFmt("%s:%s\r", strPtr(headerKey), strPtr(httpHeaderGet(requestHeader, headerKey)))); - } - } - - // Write out blank line to end the headers - ioWriteLine(tlsSessionIoWrite(this->tlsSession), CR_BUF); - - // Write out body if any - if (body != NULL) - ioWrite(tlsSessionIoWrite(this->tlsSession), body); - - // Flush all writes - ioWriteFlush(tlsSessionIoWrite(this->tlsSession)); - - // Read status - String *status = ioReadLine(tlsSessionIoRead(this->tlsSession)); - - // Check status ends with a CR and remove it to make error formatting easier and more accurate - if (!strEndsWith(status, CR_STR)) - THROW_FMT(FormatError, "HTTP response status '%s' should be CR-terminated", strPtr(status)); - - status = strSubN(status, 0, strSize(status) - 1); - - // Check status is at least the minimum required length to avoid harder to interpret errors later on - if (strSize(status) < sizeof(HTTP_VERSION) + 4) - THROW_FMT(FormatError, "HTTP response '%s' has invalid length", strPtr(strTrim(status))); - - // Check status starts with the correct http version - if (!strBeginsWith(status, HTTP_VERSION_STR)) - THROW_FMT(FormatError, "HTTP version of response '%s' must be " HTTP_VERSION, strPtr(status)); - - // Read status code - status = strSub(status, sizeof(HTTP_VERSION)); - - int spacePos = strChr(status, ' '); - - if (spacePos != 3) - THROW_FMT(FormatError, "response status '%s' must have a space after the status code", strPtr(status)); - - this->responseCode = cvtZToUInt(strPtr(strSubN(status, 0, (size_t)spacePos))); - - // Read reason phrase. A missing reason phrase will be represented as an empty string. - MEM_CONTEXT_BEGIN(this->memContext) - { - this->responseMessage = strSub(status, (size_t)spacePos + 1); - } - MEM_CONTEXT_END(); - - // Read headers - MEM_CONTEXT_BEGIN(this->memContext) - { - this->responseHeader = httpHeaderNew(NULL); - } - MEM_CONTEXT_END(); - - do - { - // Read the next header - String *header = strTrim(ioReadLine(tlsSessionIoRead(this->tlsSession))); - - // If the header is empty then we have reached the end of the headers - if (strSize(header) == 0) - break; - - // Split the header and store it - int colonPos = strChr(header, ':'); - - if (colonPos < 0) - THROW_FMT(FormatError, "header '%s' missing colon", strPtr(strTrim(header))); - - String *headerKey = strLower(strTrim(strSubN(header, 0, (size_t)colonPos))); - String *headerValue = strTrim(strSub(header, (size_t)colonPos + 1)); - - httpHeaderAdd(this->responseHeader, headerKey, headerValue); - - // Read transfer encoding (only chunked is supported) - if (strEq(headerKey, HTTP_HEADER_TRANSFER_ENCODING_STR)) - { - // Error if transfer encoding is not chunked - if (!strEq(headerValue, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR)) - { - THROW_FMT( - FormatError, "only '%s' is supported for '%s' header", HTTP_VALUE_TRANSFER_ENCODING_CHUNKED, - HTTP_HEADER_TRANSFER_ENCODING); - } - - this->contentChunked = true; - } - - // Read content size - if (strEq(headerKey, HTTP_HEADER_CONTENT_LENGTH_STR)) - { - this->contentSize = cvtZToUInt64(strPtr(headerValue)); - this->contentRemaining = this->contentSize; - } - - // If the server notified of a closed connection then close the client connection after reading content. This - // prevents doing a retry on the next request when using the closed connection. - if (strEq(headerKey, HTTP_HEADER_CONNECTION_STR) && strEq(headerValue, HTTP_VALUE_CONNECTION_CLOSE_STR)) - { - this->closeOnContentEof = true; - httpClientStatLocal.close++; - } - } - while (1); - - // Error if transfer encoding and content length are both set - if (this->contentChunked && this->contentSize > 0) - { - THROW_FMT( - FormatError, "'%s' and '%s' headers are both set", HTTP_HEADER_TRANSFER_ENCODING, - HTTP_HEADER_CONTENT_LENGTH); - } - - // Was content returned in the response? HEAD will report content but not actually return any. - bool contentExists = - (this->contentChunked || this->contentSize > 0 || this->closeOnContentEof) && !strEq(verb, HTTP_VERB_HEAD_STR); - this->contentEof = !contentExists; - - // If all content should be returned from this function then read the buffer. Also read the response if there has - // been an error. - if (returnContent || !httpClientResponseCodeOk(this)) - { - if (contentExists) - { - result = bufNew(0); - - do - { - bufResize(result, bufSize(result) + ioBufferSize()); - httpClientRead(this, result, true); - } - while (!httpClientEof(this)); - } - } - // Else create an io object, even if there is no content. This makes the logic for readers easier -- they can just - // check eof rather than also checking if the io object exists. - else - { - MEM_CONTEXT_BEGIN(this->memContext) - { - this->ioRead = ioReadNewP(this, .eof = httpClientEof, .read = httpClientRead); - ioReadOpen(this->ioRead); - } - MEM_CONTEXT_END(); - } - - // If the server notified that it would close the connection and there is no content then close the client side - if (this->closeOnContentEof && !contentExists) - { - tlsSessionFree(this->tlsSession); - this->tlsSession = NULL; - } - - // Retry when response code is 5xx. These errors generally represent a server error for a request that looks valid. - // There are a few errors that might be permanently fatal but they are rare and it seems best not to try and pick - // and choose errors in this class to retry. - if (httpClientResponseCode(this) / 100 == HTTP_RESPONSE_CODE_RETRY_CLASS) - THROW_FMT(ServiceError, "[%u] %s", httpClientResponseCode(this), strPtr(httpClientResponseMessage(this))); - } - CATCH_ANY() - { - tlsSessionFree(this->tlsSession); - this->tlsSession = NULL; - - // Retry if wait time has not expired - if (waitMore(wait)) - { - LOG_DEBUG_FMT("retry %s: %s", errorTypeName(errorType()), errorMessage()); - retry = true; - - httpClientStatLocal.retry++; - } - else - RETHROW(); - } - TRY_END(); - } - while (retry); - - // Move the result buffer (if any) to the parent context - bufMove(result, memContextPrior()); - - httpClientStatLocal.request++; + // Move session to the calling context + httpSessionMove(result, memContextCurrent()); + } + // Else create a new session + else + { + result = httpSessionNew(this, tlsClientOpen(this->tlsClient)); + httpClientStat.session++; } - MEM_CONTEXT_TEMP_END(); - FUNCTION_LOG_RETURN(BUFFER, result); + FUNCTION_LOG_RETURN(HTTP_SESSION, result); +} + +/**********************************************************************************************************************************/ +void +httpClientReuse(HttpClient *this, HttpSession *session) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(HTTP_CLIENT, this); + FUNCTION_LOG_PARAM(HTTP_SESSION, session); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(session != NULL); + + httpSessionMove(session, lstMemContext(this->sessionReuseList)); + lstAdd(this->sessionReuseList, &session); + + FUNCTION_LOG_RETURN_VOID(); } /**********************************************************************************************************************************/ @@ -503,118 +123,13 @@ httpClientStatStr(void) String *result = NULL; - if (httpClientStatLocal.object > 0) + if (httpClientStat.object > 0) { result = strNewFmt( "http statistics: objects %" PRIu64 ", sessions %" PRIu64 ", requests %" PRIu64 ", retries %" PRIu64 ", closes %" PRIu64, - httpClientStatLocal.object, httpClientStatLocal.session, httpClientStatLocal.request, httpClientStatLocal.retry, - httpClientStatLocal.close); + httpClientStat.object, httpClientStat.session, httpClientStat.request, httpClientStat.retry, httpClientStat.close); } FUNCTION_TEST_RETURN(result); } - -/**********************************************************************************************************************************/ -void -httpClientDone(HttpClient *this) -{ - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(HTTP_CLIENT, this); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - - if (this->ioRead != NULL) - { - // If it looks like we were in the middle of a response then close the TLS session so we can start clean next time - if (!this->contentEof) - { - tlsSessionFree(this->tlsSession); - this->tlsSession = NULL; - } - - ioReadFree(this->ioRead); - this->ioRead = NULL; - } - - FUNCTION_LOG_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -bool -httpClientBusy(const HttpClient *this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(HTTP_CLIENT, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - FUNCTION_TEST_RETURN(this->ioRead); -} - -/**********************************************************************************************************************************/ -IoRead * -httpClientIoRead(const HttpClient *this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(HTTP_CLIENT, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - FUNCTION_TEST_RETURN(this->ioRead); -} - -/**********************************************************************************************************************************/ -unsigned int -httpClientResponseCode(const HttpClient *this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(HTTP_CLIENT, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - FUNCTION_TEST_RETURN(this->responseCode); -} - -/**********************************************************************************************************************************/ -bool -httpClientResponseCodeOk(const HttpClient *this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(HTTP_CLIENT, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - FUNCTION_TEST_RETURN(this->responseCode / 100 == 2); -} - -/**********************************************************************************************************************************/ -const HttpHeader * -httpClientResponseHeader(const HttpClient *this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(HTTP_CLIENT, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - FUNCTION_TEST_RETURN(this->responseHeader); -} - -/**********************************************************************************************************************************/ -const String * -httpClientResponseMessage(const HttpClient *this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(HTTP_CLIENT, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - FUNCTION_TEST_RETURN(this->responseMessage); -} diff --git a/src/common/io/http/client.h b/src/common/io/http/client.h index 3dc5c4b4b..ef0d94ae6 100644 --- a/src/common/io/http/client.h +++ b/src/common/io/http/client.h @@ -4,9 +4,14 @@ HTTP Client A robust HTTP client with connection reuse and automatic retries. Using a single object to make multiple requests is more efficient because connections are reused whenever possible. Requests are -automatically retried when the connection has been closed by the server. Any 5xx response is also retried. +automatically retried when the connection has been closed by the server. Any 5xx response is also retried. Only the HTTPS protocol is currently supported. + +IMPORTANT NOTE: HttpClient should have a longer lifetime than any active HttpSession objects. This does not apply to HttpSession +objects that are freed, i.e. if an error occurs it does not matter in what order HttpClient and HttpSession objects are destroyed, +or HttpSession objects that have been returned to the client with httpClientReuse(). The danger is when an active HttpResponse +completes and tries to call httpClientReuse() on an HttpClient that has been freed thus causing a segfault. ***********************************************************************************************************************************/ #ifndef COMMON_IO_HTTP_CLIENT_H #define COMMON_IO_HTTP_CLIENT_H @@ -19,41 +24,8 @@ Object type typedef struct HttpClient HttpClient; -#include "common/io/http/header.h" -#include "common/io/http/query.h" -#include "common/io/read.h" +#include "common/io/http/session.h" #include "common/time.h" -#include "common/type/stringList.h" - -/*********************************************************************************************************************************** -HTTP Constants -***********************************************************************************************************************************/ -#define HTTP_VERB_DELETE "DELETE" - STRING_DECLARE(HTTP_VERB_DELETE_STR); -#define HTTP_VERB_GET "GET" - STRING_DECLARE(HTTP_VERB_GET_STR); -#define HTTP_VERB_HEAD "HEAD" - STRING_DECLARE(HTTP_VERB_HEAD_STR); -#define HTTP_VERB_POST "POST" - STRING_DECLARE(HTTP_VERB_POST_STR); -#define HTTP_VERB_PUT "PUT" - STRING_DECLARE(HTTP_VERB_PUT_STR); - -#define HTTP_HEADER_AUTHORIZATION "authorization" - STRING_DECLARE(HTTP_HEADER_AUTHORIZATION_STR); -#define HTTP_HEADER_CONTENT_LENGTH "content-length" - STRING_DECLARE(HTTP_HEADER_CONTENT_LENGTH_STR); -#define HTTP_HEADER_CONTENT_MD5 "content-md5" - STRING_DECLARE(HTTP_HEADER_CONTENT_MD5_STR); -#define HTTP_HEADER_ETAG "etag" - STRING_DECLARE(HTTP_HEADER_ETAG_STR); -#define HTTP_HEADER_HOST "host" - STRING_DECLARE(HTTP_HEADER_HOST_STR); -#define HTTP_HEADER_LAST_MODIFIED "last-modified" - STRING_DECLARE(HTTP_HEADER_LAST_MODIFIED_STR); - -#define HTTP_RESPONSE_CODE_FORBIDDEN 403 -#define HTTP_RESPONSE_CODE_NOT_FOUND 404 /*********************************************************************************************************************************** Statistics @@ -62,11 +34,13 @@ typedef struct HttpClientStat { uint64_t object; // Objects created uint64_t session; // TLS sessions created - uint64_t request; // Requests (i.e. calls to httpClientRequest()) + uint64_t request; // Requests (i.e. calls to httpRequestNew()) uint64_t retry; // Request retries uint64_t close; // Closes forced by server } HttpClientStat; +extern HttpClientStat httpClientStat; + /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ @@ -76,19 +50,11 @@ HttpClient *httpClientNew( /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ -// Is the HTTP object busy? -bool httpClientBusy(const HttpClient *this); +// Open a new session +HttpSession *httpClientOpen(HttpClient *this); -// Mark the client as done if read is complete -void httpClientDone(HttpClient *this); - -// Perform a request -Buffer *httpClientRequest( - HttpClient *this, const String *verb, const String *uri, const HttpQuery *query, const HttpHeader *requestHeader, - const Buffer *body, bool returnContent); - -// Is this response code OK, i.e. 2XX? -bool httpClientResponseCodeOk(const HttpClient *this); +// Request/response finished cleanly so session can be reused +void httpClientReuse(HttpClient *this, HttpSession *session); // Format statistics to a string String *httpClientStatStr(void); @@ -96,22 +62,7 @@ String *httpClientStatStr(void); /*********************************************************************************************************************************** Getters/Setters ***********************************************************************************************************************************/ -// Read interface -IoRead *httpClientIoRead(const HttpClient *this); - -// Get the response code -unsigned int httpClientResponseCode(const HttpClient *this); - -// Response headers -const HttpHeader *httpClientResponseHeader(const HttpClient *this); - -// Response message -const String *httpClientResponseMessage(const HttpClient *this); - -/*********************************************************************************************************************************** -Destructor -***********************************************************************************************************************************/ -void httpClientFree(HttpClient *this); +TimeMSec httpClientTimeout(const HttpClient *this); /*********************************************************************************************************************************** Macros for function logging diff --git a/src/common/io/http/query.c b/src/common/io/http/query.c index 5910e67b6..bb79bc6c2 100644 --- a/src/common/io/http/query.c +++ b/src/common/io/http/query.c @@ -46,6 +46,35 @@ httpQueryNew(void) FUNCTION_TEST_RETURN(this); } +/**********************************************************************************************************************************/ +HttpQuery * +httpQueryDup(const HttpQuery *query) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(HTTP_QUERY, query); + FUNCTION_TEST_END(); + + HttpQuery *this = NULL; + + if (query != NULL) + { + MEM_CONTEXT_NEW_BEGIN("HttpQuery") + { + // Allocate state and set context + this = memNew(sizeof(HttpQuery)); + + *this = (HttpQuery) + { + .memContext = MEM_CONTEXT_NEW(), + .kv = kvDup(query->kv), + }; + } + MEM_CONTEXT_NEW_END(); + } + + FUNCTION_TEST_RETURN(this); +} + /**********************************************************************************************************************************/ HttpQuery * httpQueryAdd(HttpQuery *this, const String *key, const String *value) diff --git a/src/common/io/http/query.h b/src/common/io/http/query.h index abae0ab8b..0cd4245b6 100644 --- a/src/common/io/http/query.h +++ b/src/common/io/http/query.h @@ -20,6 +20,7 @@ typedef struct HttpQuery HttpQuery; Constructors ***********************************************************************************************************************************/ HttpQuery *httpQueryNew(void); +HttpQuery *httpQueryDup(const HttpQuery *query); /*********************************************************************************************************************************** Functions diff --git a/src/common/io/http/request.c b/src/common/io/http/request.c new file mode 100644 index 000000000..7cc39e234 --- /dev/null +++ b/src/common/io/http/request.c @@ -0,0 +1,313 @@ +/*********************************************************************************************************************************** +HTTP Request +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include "common/debug.h" +#include "common/io/http/common.h" +#include "common/io/http/request.h" +#include "common/log.h" +#include "common/type/object.h" +#include "common/wait.h" + +/*********************************************************************************************************************************** +HTTP constants +***********************************************************************************************************************************/ +STRING_EXTERN(HTTP_VERSION_STR, HTTP_VERSION); + +STRING_EXTERN(HTTP_VERB_DELETE_STR, HTTP_VERB_DELETE); +STRING_EXTERN(HTTP_VERB_GET_STR, HTTP_VERB_GET); +STRING_EXTERN(HTTP_VERB_HEAD_STR, HTTP_VERB_HEAD); +STRING_EXTERN(HTTP_VERB_POST_STR, HTTP_VERB_POST); +STRING_EXTERN(HTTP_VERB_PUT_STR, HTTP_VERB_PUT); + +STRING_EXTERN(HTTP_HEADER_AUTHORIZATION_STR, HTTP_HEADER_AUTHORIZATION); +STRING_EXTERN(HTTP_HEADER_CONTENT_LENGTH_STR, HTTP_HEADER_CONTENT_LENGTH); +STRING_EXTERN(HTTP_HEADER_CONTENT_MD5_STR, HTTP_HEADER_CONTENT_MD5); +STRING_EXTERN(HTTP_HEADER_ETAG_STR, HTTP_HEADER_ETAG); +STRING_EXTERN(HTTP_HEADER_HOST_STR, HTTP_HEADER_HOST); +STRING_EXTERN(HTTP_HEADER_LAST_MODIFIED_STR, HTTP_HEADER_LAST_MODIFIED); + +// 5xx errors that should always be retried +#define HTTP_RESPONSE_CODE_RETRY_CLASS 5 + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct HttpRequest +{ + MemContext *memContext; // Mem context + HttpClient *client; // HTTP client + const String *verb; // HTTP verb (GET, POST, etc.) + const String *uri; // HTTP URI + const HttpQuery *query; // HTTP query + const HttpHeader *header; // HTTP headers + const Buffer *content; // HTTP content + + HttpSession *session; // Session for async requests +}; + +OBJECT_DEFINE_MOVE(HTTP_REQUEST); +OBJECT_DEFINE_FREE(HTTP_REQUEST); + +OBJECT_DEFINE_GET(Verb, const, HTTP_REQUEST, const String *, verb); +OBJECT_DEFINE_GET(Uri, const, HTTP_REQUEST, const String *, uri); +OBJECT_DEFINE_GET(Query, const, HTTP_REQUEST, const HttpQuery *, query); +OBJECT_DEFINE_GET(Header, const, HTTP_REQUEST, const HttpHeader *, header); + +/*********************************************************************************************************************************** +Process the request +***********************************************************************************************************************************/ +static HttpResponse * +httpRequestProcess(HttpRequest *this, bool requestOnly, bool contentCache) +{ + FUNCTION_LOG_BEGIN(logLevelDebug) + FUNCTION_LOG_PARAM(HTTP_REQUEST, this); + FUNCTION_LOG_PARAM(BOOL, requestOnly); + FUNCTION_LOG_PARAM(BOOL, contentCache); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + // HTTP Response + HttpResponse *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + bool retry; + Wait *wait = waitNew(httpClientTimeout(this->client)); + + do + { + // Assume there will be no retry + retry = false; + + TRY_BEGIN() + { + MEM_CONTEXT_TEMP_BEGIN() + { + HttpSession *session = NULL; + + // If a session is saved then the request was already successfully sent + if (this->session != NULL) + { + session = httpSessionMove(this->session, memContextCurrent()); + this->session = NULL; + } + // Else the request has not been sent yet or this is a retry + else + { + session = httpClientOpen(this->client); + + // Write the request + String *queryStr = httpQueryRender(this->query); + + ioWriteStrLine( + httpSessionIoWrite(session), + strNewFmt( + "%s %s%s%s " HTTP_VERSION "\r", strPtr(this->verb), strPtr(httpUriEncode(this->uri, true)), + queryStr == NULL ? "" : "?", queryStr == NULL ? "" : strPtr(queryStr))); + + // Write headers + const StringList *headerList = httpHeaderList(this->header); + + for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++) + { + const String *headerKey = strLstGet(headerList, headerIdx); + + ioWriteStrLine( + httpSessionIoWrite(session), + strNewFmt("%s:%s\r", strPtr(headerKey), strPtr(httpHeaderGet(this->header, headerKey)))); + } + + // Write out blank line to end the headers + ioWriteLine(httpSessionIoWrite(session), CR_BUF); + + // Write out content if any + if (this->content != NULL) + ioWrite(httpSessionIoWrite(session), this->content); + + // Flush all writes + ioWriteFlush(httpSessionIoWrite(session)); + + // If only performing the request then move the session to the object context + if (requestOnly) + this->session = httpSessionMove(session, this->memContext); + } + + // Wait for response + if (!requestOnly) + { + result = httpResponseNew(session, this->verb, contentCache); + + // Retry when response code is 5xx. These errors generally represent a server error for a request that + // looks valid. There are a few errors that might be permanently fatal but they are rare and it seems best + // not to try and pick and choose errors in this class to retry. + if (httpResponseCode(result) / 100 == HTTP_RESPONSE_CODE_RETRY_CLASS) + THROW_FMT(ServiceError, "[%u] %s", httpResponseCode(result), strPtr(httpResponseReason(result))); + + // Move response to outer temp context + httpResponseMove(result, memContextPrior()); + } + } + MEM_CONTEXT_TEMP_END(); + } + CATCH_ANY() + { + // Retry if wait time has not expired + if (waitMore(wait)) + { + LOG_DEBUG_FMT("retry %s: %s", errorTypeName(errorType()), errorMessage()); + retry = true; + + httpClientStat.retry++; + } + else + RETHROW(); + } + TRY_END(); + } + while (retry); + + // Move response to calling context + httpResponseMove(result, memContextPrior()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(HTTP_RESPONSE, result); +} + +/**********************************************************************************************************************************/ +HttpRequest * +httpRequestNew(HttpClient *client, const String *verb, const String *uri, HttpRequestNewParam param) +{ + FUNCTION_LOG_BEGIN(logLevelDebug) + FUNCTION_LOG_PARAM(HTTP_CLIENT, client); + FUNCTION_LOG_PARAM(STRING, verb); + FUNCTION_LOG_PARAM(STRING, uri); + FUNCTION_LOG_PARAM(HTTP_QUERY, param.query); + FUNCTION_LOG_PARAM(HTTP_HEADER, param.header); + FUNCTION_LOG_PARAM(BUFFER, param.content); + FUNCTION_LOG_END(); + + ASSERT(verb != NULL); + ASSERT(uri != NULL); + + HttpRequest *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("HttpRequest") + { + this = memNew(sizeof(HttpRequest)); + + *this = (HttpRequest) + { + .memContext = MEM_CONTEXT_NEW(), + .client = client, + .verb = strDup(verb), + .uri = strDup(uri), + .query = httpQueryDup(param.query), + .header = param.header == NULL ? httpHeaderNew(NULL) : httpHeaderDup(param.header, NULL), + .content = param.content == NULL ? NULL : bufDup(param.content), + }; + + // Send the request + httpRequestProcess(this, true, false); + httpClientStat.request++; + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_LOG_RETURN(HTTP_REQUEST, this); +} + +/**********************************************************************************************************************************/ +HttpResponse * +httpRequest(HttpRequest *this, bool contentCache) +{ + FUNCTION_LOG_BEGIN(logLevelDebug) + FUNCTION_LOG_PARAM(HTTP_REQUEST, this); + FUNCTION_LOG_PARAM(BOOL, contentCache); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + FUNCTION_LOG_RETURN(HTTP_RESPONSE, httpRequestProcess(this, false, contentCache)); +} + +/**********************************************************************************************************************************/ +void +httpRequestError(const HttpRequest *this, HttpResponse *response) +{ + FUNCTION_LOG_BEGIN(logLevelTrace) + FUNCTION_LOG_PARAM(HTTP_REQUEST, this); + FUNCTION_LOG_PARAM(HTTP_RESPONSE, response); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(response != NULL); + + // Error code + String *error = strNewFmt("HTTP request failed with %u", httpResponseCode(response)); + + // Add reason when present + if (strSize(httpResponseReason(response)) > 0) + strCatFmt(error, " (%s)", strPtr(httpResponseReason(response))); + + // Output uri/query + strCatZ(error, ":\n*** URI/Query ***:"); + + strCatFmt(error, "\n%s", strPtr(httpUriEncode(this->uri, true))); + + if (this->query != NULL) + strCatFmt(error, "?%s", strPtr(httpQueryRender(this->query))); + + // Output request headers + const StringList *requestHeaderList = httpHeaderList(this->header); + + if (strLstSize(requestHeaderList) > 0) + { + strCatZ(error, "\n*** Request Headers ***:"); + + for (unsigned int requestHeaderIdx = 0; requestHeaderIdx < strLstSize(requestHeaderList); requestHeaderIdx++) + { + const String *key = strLstGet(requestHeaderList, requestHeaderIdx); + + strCatFmt( + error, "\n%s: %s", strPtr(key), + httpHeaderRedact(this->header, key) ? "" : strPtr(httpHeaderGet(this->header, key))); + } + } + + // Output response headers + const HttpHeader *responseHeader = httpResponseHeader(response); + const StringList *responseHeaderList = httpHeaderList(responseHeader); + + if (strLstSize(responseHeaderList) > 0) + { + strCatZ(error, "\n*** Response Headers ***:"); + + for (unsigned int responseHeaderIdx = 0; responseHeaderIdx < strLstSize(responseHeaderList); responseHeaderIdx++) + { + const String *key = strLstGet(responseHeaderList, responseHeaderIdx); + strCatFmt(error, "\n%s: %s", strPtr(key), strPtr(httpHeaderGet(responseHeader, key))); + } + } + + // Add response content, if any + if (bufUsed(httpResponseContent(response)) > 0) + { + strCatZ(error, "\n*** Response Content ***:\n"); + strCat(error, strNewBuf(httpResponseContent(response))); + } + + THROW(ProtocolError, strPtr(error)); +} + +/**********************************************************************************************************************************/ +String * +httpRequestToLog(const HttpRequest *this) +{ + return strNewFmt( + "{verb: %s, uri: %s, query: %s, header: %s, contentSize: %zu", + strPtr(this->verb), strPtr(this->uri), this->query == NULL ? "null" : strPtr(httpQueryToLog(this->query)), + strPtr(httpHeaderToLog(this->header)), this->content == NULL ? 0 : bufUsed(this->content)); +} diff --git a/src/common/io/http/request.h b/src/common/io/http/request.h new file mode 100644 index 000000000..5f4521233 --- /dev/null +++ b/src/common/io/http/request.h @@ -0,0 +1,111 @@ +/*********************************************************************************************************************************** +HTTP Request + +Send a request to an HTTP server and get a response. The interface is natively asynchronous, i.e. httpRequestNew() sends a request +and httpRequest() waits for a response. These can be called together for synchronous behavior or separately for asynchronous +behavior. +***********************************************************************************************************************************/ +#ifndef COMMON_IO_HTTP_REQUEST_H +#define COMMON_IO_HTTP_REQUEST_H + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +#define HTTP_REQUEST_TYPE HttpRequest +#define HTTP_REQUEST_PREFIX httpRequest + +typedef struct HttpRequest HttpRequest; + +#include "common/io/http/header.h" +#include "common/io/http/query.h" +#include "common/io/http/response.h" + +/*********************************************************************************************************************************** +HTTP Constants +***********************************************************************************************************************************/ +#define HTTP_VERSION "HTTP/1.1" + STRING_DECLARE(HTTP_VERSION_STR); + +#define HTTP_VERB_DELETE "DELETE" + STRING_DECLARE(HTTP_VERB_DELETE_STR); +#define HTTP_VERB_GET "GET" + STRING_DECLARE(HTTP_VERB_GET_STR); +#define HTTP_VERB_HEAD "HEAD" + STRING_DECLARE(HTTP_VERB_HEAD_STR); +#define HTTP_VERB_POST "POST" + STRING_DECLARE(HTTP_VERB_POST_STR); +#define HTTP_VERB_PUT "PUT" + STRING_DECLARE(HTTP_VERB_PUT_STR); + +#define HTTP_HEADER_AUTHORIZATION "authorization" + STRING_DECLARE(HTTP_HEADER_AUTHORIZATION_STR); +#define HTTP_HEADER_CONTENT_LENGTH "content-length" + STRING_DECLARE(HTTP_HEADER_CONTENT_LENGTH_STR); +#define HTTP_HEADER_CONTENT_MD5 "content-md5" + STRING_DECLARE(HTTP_HEADER_CONTENT_MD5_STR); +#define HTTP_HEADER_ETAG "etag" + STRING_DECLARE(HTTP_HEADER_ETAG_STR); +#define HTTP_HEADER_HOST "host" + STRING_DECLARE(HTTP_HEADER_HOST_STR); +#define HTTP_HEADER_LAST_MODIFIED "last-modified" + STRING_DECLARE(HTTP_HEADER_LAST_MODIFIED_STR); + +/*********************************************************************************************************************************** +Constructors +***********************************************************************************************************************************/ +typedef struct HttpRequestNewParam +{ + VAR_PARAM_HEADER; + const HttpQuery *query; + const HttpHeader *header; + const Buffer *content; +} HttpRequestNewParam; + +#define httpRequestNewP(client, verb, uri, ...) \ + httpRequestNew(client, verb, uri, (HttpRequestNewParam){VAR_PARAM_INIT, __VA_ARGS__}) + +HttpRequest *httpRequestNew(HttpClient *client, const String *verb, const String *uri, HttpRequestNewParam param); + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Send a request to the server +HttpResponse *httpRequest(HttpRequest *this, bool contentCache); + +// Throw an error if the request failed +void httpRequestError(const HttpRequest *this, HttpResponse *response) __attribute__((__noreturn__)); + +// Move to a new parent mem context +HttpRequest *httpRequestMove(HttpRequest *this, MemContext *parentNew); + +/*********************************************************************************************************************************** +Getters/Setters +***********************************************************************************************************************************/ +// Request verb +const String *httpRequestVerb(const HttpRequest *this); + +// Request URI +const String *httpRequestUri(const HttpRequest *this); + +// Request query +const HttpQuery *httpRequestQuery(const HttpRequest *this); + +// Request headers +const HttpHeader *httpRequestHeader(const HttpRequest *this); + +/*********************************************************************************************************************************** +Destructor +***********************************************************************************************************************************/ +void httpRequestFree(HttpRequest *this); + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +String *httpRequestToLog(const HttpRequest *this); + +#define FUNCTION_LOG_HTTP_REQUEST_TYPE \ + HttpRequest * +#define FUNCTION_LOG_HTTP_REQUEST_FORMAT(value, buffer, bufferSize) \ + FUNCTION_LOG_STRING_OBJECT_FORMAT(value, httpRequestToLog, buffer, bufferSize) + +#endif diff --git a/src/common/io/http/response.c b/src/common/io/http/response.c new file mode 100644 index 000000000..aaa2f2006 --- /dev/null +++ b/src/common/io/http/response.c @@ -0,0 +1,415 @@ +/*********************************************************************************************************************************** +HTTP Response +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include "common/debug.h" +#include "common/io/http/client.h" +#include "common/io/http/common.h" +#include "common/io/http/request.h" +#include "common/io/http/response.h" +#include "common/io/io.h" +#include "common/io/read.intern.h" +#include "common/io/tls/client.h" +#include "common/log.h" +#include "common/type/object.h" +#include "common/wait.h" + +/*********************************************************************************************************************************** +HTTP constants +***********************************************************************************************************************************/ +#define HTTP_HEADER_CONNECTION "connection" + STRING_STATIC(HTTP_HEADER_CONNECTION_STR, HTTP_HEADER_CONNECTION); +#define HTTP_HEADER_TRANSFER_ENCODING "transfer-encoding" + STRING_STATIC(HTTP_HEADER_TRANSFER_ENCODING_STR, HTTP_HEADER_TRANSFER_ENCODING); + +#define HTTP_VALUE_CONNECTION_CLOSE "close" + STRING_STATIC(HTTP_VALUE_CONNECTION_CLOSE_STR, HTTP_VALUE_CONNECTION_CLOSE); +#define HTTP_VALUE_TRANSFER_ENCODING_CHUNKED "chunked" + STRING_STATIC(HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED); + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct HttpResponse +{ + MemContext *memContext; // Mem context + + HttpSession *session; // HTTP session + IoRead *contentRead; // Read interface for response content + + unsigned int code; // Response code (e.g. 200, 404) + String *reason; // Response reason e.g. (OK, Not Found) + HttpHeader *header; // Response headers + + bool contentChunked; // Is the response content chunked? + uint64_t contentSize; // Content size (ignored for chunked) + uint64_t contentRemaining; // Content remaining (per chunk if chunked) + bool closeOnContentEof; // Will server close after content is sent? + bool contentExists; // Does content exist? + bool contentEof; // Has all content been read? + Buffer *content; // Caches content once requested +}; + +OBJECT_DEFINE_MOVE(HTTP_RESPONSE); +OBJECT_DEFINE_FREE(HTTP_RESPONSE); + +OBJECT_DEFINE_GET(IoRead, , HTTP_RESPONSE, IoRead *, contentRead); +OBJECT_DEFINE_GET(Code, const, HTTP_RESPONSE, unsigned int, code); +OBJECT_DEFINE_GET(Header, const, HTTP_RESPONSE, const HttpHeader *, header); +OBJECT_DEFINE_GET(Reason, const, HTTP_RESPONSE, const String *, reason); + +/*********************************************************************************************************************************** +When response is done close/reuse the connection +***********************************************************************************************************************************/ +static void +httpResponseDone(HttpResponse *this) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(HTTP_RESPONSE, this); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(this->session != NULL); + + // If close was requested by the server then free the session + if (this->closeOnContentEof) + { + httpSessionFree(this->session); + + // Only update the close stats after a successful response so it is not counted if there was an error/retry + httpClientStat.close++; + } + // Else return it to the client so it can be reused + else + httpSessionDone(this->session); + + this->session = NULL; + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Read content +***********************************************************************************************************************************/ +static size_t +httpResponseRead(THIS_VOID, Buffer *buffer, bool block) +{ + THIS(HttpResponse); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(HTTP_RESPONSE, this); + FUNCTION_LOG_PARAM(BUFFER, buffer); + FUNCTION_LOG_PARAM(BOOL, block); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(buffer != NULL); + ASSERT(!bufFull(buffer)); + ASSERT(this->contentEof || this->session != NULL); + + // Read if EOF has not been reached + size_t actualBytes = 0; + + if (!this->contentEof) + { + MEM_CONTEXT_TEMP_BEGIN() + { + IoRead *rawRead = httpSessionIoRead(this->session); + + // If close was requested and no content specified then the server may send content up until the eof + if (this->closeOnContentEof && !this->contentChunked && this->contentSize == 0) + { + ioRead(rawRead, buffer); + this->contentEof = ioReadEof(rawRead); + } + // Else read using specified encoding or size + else + { + do + { + // If chunked content and no content remaining + if (this->contentChunked && this->contentRemaining == 0) + { + // Read length of next chunk + this->contentRemaining = cvtZToUInt64Base(strPtr(strTrim(ioReadLine(rawRead))), 16); + + // If content remaining is still zero then eof + if (this->contentRemaining == 0) + this->contentEof = true; + } + + // Read if there is content remaining + if (this->contentRemaining > 0) + { + // If the buffer is larger than the content that needs to be read then limit the buffer size so the read + // won't block or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is + // defined as less than 2^32 so content remaining can't be more than that. + if (bufRemains(buffer) > this->contentRemaining) + bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining)); + + actualBytes = bufRemains(buffer); + this->contentRemaining -= ioRead(rawRead, buffer); + + // Error if EOF but content read is not complete + if (ioReadEof(rawRead)) + THROW(FileReadError, "unexpected EOF reading HTTP content"); + + // Clear limit (this works even if the limit was not set and it is easier than checking) + bufLimitClear(buffer); + } + + // If no content remaining + if (this->contentRemaining == 0) + { + // If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back + // around to check. + if (this->contentChunked) + { + ioReadLine(rawRead); + } + // If total content size was provided then this is eof + else + this->contentEof = true; + } + } + while (!bufFull(buffer) && !this->contentEof); + } + + // If all content has been read + if (this->contentEof) + httpResponseDone(this); + } + MEM_CONTEXT_TEMP_END(); + } + + FUNCTION_LOG_RETURN(SIZE, (size_t)actualBytes); +} + +/*********************************************************************************************************************************** +Has all content been read? +***********************************************************************************************************************************/ +static bool +httpResponseEof(THIS_VOID) +{ + THIS(HttpResponse); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(HTTP_RESPONSE, this); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + FUNCTION_LOG_RETURN(BOOL, this->contentEof); +} + +/**********************************************************************************************************************************/ +HttpResponse * +httpResponseNew(HttpSession *session, const String *verb, bool contentCache) +{ + FUNCTION_LOG_BEGIN(logLevelDebug) + FUNCTION_LOG_PARAM(HTTP_SESSION, session); + FUNCTION_LOG_PARAM(STRING, verb); + FUNCTION_LOG_PARAM(BOOL, contentCache); + FUNCTION_LOG_END(); + + ASSERT(session != NULL); + ASSERT(verb != NULL); + + HttpResponse *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("HttpResponse") + { + this = memNew(sizeof(HttpResponse)); + + *this = (HttpResponse) + { + .memContext = MEM_CONTEXT_NEW(), + .session = httpSessionMove(session, memContextCurrent()), + .header = httpHeaderNew(NULL), + }; + + MEM_CONTEXT_TEMP_BEGIN() + { + // Read status + String *status = ioReadLine(httpSessionIoRead(this->session)); + + // Check status ends with a CR and remove it to make error formatting easier and more accurate + if (!strEndsWith(status, CR_STR)) + THROW_FMT(FormatError, "HTTP response status '%s' should be CR-terminated", strPtr(status)); + + status = strSubN(status, 0, strSize(status) - 1); + + // Check status is at least the minimum required length to avoid harder to interpret errors later on + if (strSize(status) < sizeof(HTTP_VERSION) + 4) + THROW_FMT(FormatError, "HTTP response '%s' has invalid length", strPtr(strTrim(status))); + + // Check status starts with the correct http version + if (!strBeginsWith(status, HTTP_VERSION_STR)) + THROW_FMT(FormatError, "HTTP version of response '%s' must be " HTTP_VERSION, strPtr(status)); + + // Read status code + status = strSub(status, sizeof(HTTP_VERSION)); + + int spacePos = strChr(status, ' '); + + if (spacePos != 3) + THROW_FMT(FormatError, "response status '%s' must have a space after the status code", strPtr(status)); + + this->code = cvtZToUInt(strPtr(strSubN(status, 0, (size_t)spacePos))); + + // Read reason phrase. A missing reason phrase will be represented as an empty string. + MEM_CONTEXT_BEGIN(this->memContext) + { + this->reason = strSub(status, (size_t)spacePos + 1); + } + MEM_CONTEXT_END(); + + // Read headers + do + { + // Read the next header + String *header = strTrim(ioReadLine(httpSessionIoRead(this->session))); + + // If the header is empty then we have reached the end of the headers + if (strSize(header) == 0) + break; + + // Split the header and store it + int colonPos = strChr(header, ':'); + + if (colonPos < 0) + THROW_FMT(FormatError, "header '%s' missing colon", strPtr(strTrim(header))); + + String *headerKey = strLower(strTrim(strSubN(header, 0, (size_t)colonPos))); + String *headerValue = strTrim(strSub(header, (size_t)colonPos + 1)); + + httpHeaderAdd(this->header, headerKey, headerValue); + + // Read transfer encoding (only chunked is supported) + if (strEq(headerKey, HTTP_HEADER_TRANSFER_ENCODING_STR)) + { + // Error if transfer encoding is not chunked + if (!strEq(headerValue, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR)) + { + THROW_FMT( + FormatError, "only '%s' is supported for '%s' header", HTTP_VALUE_TRANSFER_ENCODING_CHUNKED, + HTTP_HEADER_TRANSFER_ENCODING); + } + + this->contentChunked = true; + } + + // Read content size + if (strEq(headerKey, HTTP_HEADER_CONTENT_LENGTH_STR)) + { + this->contentSize = cvtZToUInt64(strPtr(headerValue)); + this->contentRemaining = this->contentSize; + } + + // If the server notified of a closed connection then close the client connection after reading content. This + // prevents doing a retry on the next request when using the closed connection. + if (strEq(headerKey, HTTP_HEADER_CONNECTION_STR) && strEq(headerValue, HTTP_VALUE_CONNECTION_CLOSE_STR)) + this->closeOnContentEof = true; + + } + while (1); + + // Error if transfer encoding and content length are both set + if (this->contentChunked && this->contentSize > 0) + { + THROW_FMT( + FormatError, "'%s' and '%s' headers are both set", HTTP_HEADER_TRANSFER_ENCODING, + HTTP_HEADER_CONTENT_LENGTH); + } + + // Was content returned in the response? HEAD will report content but not actually return any. + this->contentExists = + (this->contentChunked || this->contentSize > 0 || this->closeOnContentEof) && !strEq(verb, HTTP_VERB_HEAD_STR); + this->contentEof = !this->contentExists; + + // Create an io object, even if there is no content. This makes the logic for readers easier -- they can just check eof + // rather than also checking if the io object exists. + MEM_CONTEXT_BEGIN(this->memContext) + { + this->contentRead = ioReadNewP(this, .eof = httpResponseEof, .read = httpResponseRead); + ioReadOpen(this->contentRead); + } + MEM_CONTEXT_END(); + + // If there is no content then we are done with the client + if (!this->contentExists) + { + httpResponseDone(this); + } + // Else cache content when requested or on error + else if (contentCache || !httpResponseCodeOk(this)) + { + MEM_CONTEXT_BEGIN(this->memContext) + { + httpResponseContent(this); + } + MEM_CONTEXT_END(); + } + } + MEM_CONTEXT_TEMP_END(); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_LOG_RETURN(HTTP_RESPONSE, this); +} + +/**********************************************************************************************************************************/ +const Buffer * +httpResponseContent(HttpResponse *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(HTTP_RESPONSE, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (this->content == NULL) + { + this->content = bufNew(0); + + if (this->contentExists) + { + do + { + bufResize(this->content, bufSize(this->content) + ioBufferSize()); + httpResponseRead(this, this->content, true); + } + while (!httpResponseEof(this)); + + bufResize(this->content, bufUsed(this->content)); + } + } + + FUNCTION_TEST_RETURN(this->content); +} + +/**********************************************************************************************************************************/ +bool +httpResponseCodeOk(const HttpResponse *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(HTTP_RESPONSE, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(this->code / 100 == 2); +} + +/**********************************************************************************************************************************/ +String * +httpResponseToLog(const HttpResponse *this) +{ + return strNewFmt( + "{code: %u, reason: %s, header: %s, contentChunked: %s, contentSize: %" PRIu64 ", contentRemaining: %" PRIu64 + ", closeOnContentEof: %s, contentExists: %s, contentEof: %s, contentCached: %s", + this->code, strPtr(this->reason), strPtr(httpHeaderToLog(this->header)), + cvtBoolToConstZ(this->contentChunked), this->contentSize, this->contentRemaining, cvtBoolToConstZ(this->closeOnContentEof), + cvtBoolToConstZ(this->contentExists), cvtBoolToConstZ(this->contentEof), cvtBoolToConstZ(this->content != NULL)); +} diff --git a/src/common/io/http/response.h b/src/common/io/http/response.h new file mode 100644 index 000000000..d161f7b84 --- /dev/null +++ b/src/common/io/http/response.h @@ -0,0 +1,79 @@ +/*********************************************************************************************************************************** +HTTP Response + +Response created after a successful request. Once the content is read the underlying connection may be recycled but the headers, +cached content, etc. will still be available for the lifetime of the object. +***********************************************************************************************************************************/ +#ifndef COMMON_IO_HTTP_RESPONSE_H +#define COMMON_IO_HTTP_RESPONSE_H + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +#define HTTP_RESPONSE_TYPE HttpResponse +#define HTTP_RESPONSE_PREFIX httpResponse + +typedef struct HttpResponse HttpResponse; + +#include "common/io/http/header.h" +#include "common/io/http/session.h" +#include "common/io/read.h" + +/*********************************************************************************************************************************** +HTTP Response Constants +***********************************************************************************************************************************/ +#define HTTP_RESPONSE_CODE_FORBIDDEN 403 +#define HTTP_RESPONSE_CODE_NOT_FOUND 404 + +/*********************************************************************************************************************************** +Constructors +***********************************************************************************************************************************/ +HttpResponse *httpResponseNew(HttpSession *session, const String *verb, bool contentCache); + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Is this response code OK, i.e. 2XX? +bool httpResponseCodeOk(const HttpResponse *this); + +// Fetch all response content. Content will be cached so it can be retrieved again without additional cost. +const Buffer *httpResponseContent(HttpResponse *this); + +// Move to a new parent mem context +HttpResponse *httpResponseMove(HttpResponse *this, MemContext *parentNew); + +/*********************************************************************************************************************************** +Getters/Setters +***********************************************************************************************************************************/ +// Is the response still being read? +bool httpResponseBusy(const HttpResponse *this); + +// Read interface used to get the response content. This is intended for reading content that may be very large and will not be held +// in memory all at once. If the content must be loaded completely for processing (e.g. XML) then httpResponseContent() is simpler. +IoRead *httpResponseIoRead(HttpResponse *this); + +// Response code +unsigned int httpResponseCode(const HttpResponse *this); + +// Response headers +const HttpHeader *httpResponseHeader(const HttpResponse *this); + +// Response reason +const String *httpResponseReason(const HttpResponse *this); + +/*********************************************************************************************************************************** +Destructor +***********************************************************************************************************************************/ +void httpResponseFree(HttpResponse *this); + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +String *httpResponseToLog(const HttpResponse *this); + +#define FUNCTION_LOG_HTTP_RESPONSE_TYPE \ + HttpResponse * +#define FUNCTION_LOG_HTTP_RESPONSE_FORMAT(value, buffer, bufferSize) \ + FUNCTION_LOG_STRING_OBJECT_FORMAT(value, httpResponseToLog, buffer, bufferSize) + +#endif diff --git a/src/common/io/http/session.c b/src/common/io/http/session.c new file mode 100644 index 000000000..67c99cf3b --- /dev/null +++ b/src/common/io/http/session.c @@ -0,0 +1,95 @@ +/*********************************************************************************************************************************** +HTTP Session +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include "common/debug.h" +#include "common/io/http/session.h" +#include "common/io/io.h" +#include "common/log.h" +#include "common/memContext.h" +#include "common/type/object.h" + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct HttpSession +{ + MemContext *memContext; // Mem context + HttpClient *httpClient; // HTTP client + TlsSession *tlsSession; // TLS session +}; + +OBJECT_DEFINE_MOVE(HTTP_SESSION); +OBJECT_DEFINE_FREE(HTTP_SESSION); + +/**********************************************************************************************************************************/ +HttpSession * +httpSessionNew(HttpClient *httpClient, TlsSession *tlsSession) +{ + FUNCTION_LOG_BEGIN(logLevelDebug) + FUNCTION_LOG_PARAM(HTTP_CLIENT, httpClient); + FUNCTION_LOG_PARAM(TLS_SESSION, tlsSession); + FUNCTION_LOG_END(); + + ASSERT(httpClient != NULL); + ASSERT(tlsSession != NULL); + + HttpSession *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("HttpSession") + { + this = memNew(sizeof(HttpSession)); + + *this = (HttpSession) + { + .memContext = MEM_CONTEXT_NEW(), + .httpClient = httpClient, + .tlsSession = tlsSessionMove(tlsSession, memContextCurrent()), + }; + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_LOG_RETURN(HTTP_SESSION, this); +} + +/**********************************************************************************************************************************/ +void +httpSessionDone(HttpSession *this) +{ + FUNCTION_LOG_BEGIN(logLevelDebug) + FUNCTION_LOG_PARAM(HTTP_SESSION, this); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + httpClientReuse(this->httpClient, this); + + FUNCTION_LOG_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +IoRead * +httpSessionIoRead(HttpSession *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(HTTP_SESSION, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(tlsSessionIoRead(this->tlsSession)); +} + +/**********************************************************************************************************************************/ +IoWrite * +httpSessionIoWrite(HttpSession *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(HTTP_SESSION, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(tlsSessionIoWrite(this->tlsSession)); +} diff --git a/src/common/io/http/cache.h b/src/common/io/http/session.h similarity index 54% rename from src/common/io/http/cache.h rename to src/common/io/http/session.h index 433a486ac..aca969c69 100644 --- a/src/common/io/http/cache.h +++ b/src/common/io/http/session.h @@ -1,44 +1,58 @@ /*********************************************************************************************************************************** -HTTP Client Cache +HTTP Session -Cache HTTP clients and return one that is not busy on request. +HTTP sessions are created by calling httpClientOpen(), which is currently done exclusively by the HttpRequest object. ***********************************************************************************************************************************/ -#ifndef COMMON_IO_HTTP_CLIENT_CACHE_H -#define COMMON_IO_HTTP_CLIENT_CACHE_H +#ifndef COMMON_IO_HTTP_SESSION_H +#define COMMON_IO_HTTP_SESSION_H /*********************************************************************************************************************************** Object type ***********************************************************************************************************************************/ -#define HTTP_CLIENT_CACHE_TYPE HttpClientCache -#define HTTP_CLIENT_CACHE_PREFIX httpClientCache +#define HTTP_SESSION_TYPE HttpSession +#define HTTP_SESSION_PREFIX httpSession -typedef struct HttpClientCache HttpClientCache; +typedef struct HttpSession HttpSession; +#include "common/io/read.h" #include "common/io/http/client.h" +#include "common/io/tls/session.h" +#include "common/io/write.h" /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -HttpClientCache *httpClientCacheNew( - const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath); +HttpSession *httpSessionNew(HttpClient *client, TlsSession *session); /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ -// Get an HTTP client from the cache -HttpClient *httpClientCacheGet(HttpClientCache *this); +// Move to a new parent mem context +HttpSession *httpSessionMove(HttpSession *this, MemContext *parentNew); + +// Work with the session has finished cleanly and it can be reused +void httpSessionDone(HttpSession *this); + +/*********************************************************************************************************************************** +Getters/Setters +***********************************************************************************************************************************/ +// Read interface +IoRead *httpSessionIoRead(HttpSession *this); + +// Write interface +IoWrite *httpSessionIoWrite(HttpSession *this); /*********************************************************************************************************************************** Destructor ***********************************************************************************************************************************/ -void httpClientCacheFree(HttpClientCache *this); +void httpSessionFree(HttpSession *this); /*********************************************************************************************************************************** Macros for function logging ***********************************************************************************************************************************/ -#define FUNCTION_LOG_HTTP_CLIENT_CACHE_TYPE \ - HttpClientCache * -#define FUNCTION_LOG_HTTP_CLIENT_CACHE_FORMAT(value, buffer, bufferSize) \ - objToLog(value, "HttpClientCache", buffer, bufferSize) +#define FUNCTION_LOG_HTTP_SESSION_TYPE \ + HttpSession * +#define FUNCTION_LOG_HTTP_SESSION_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "HttpSession", buffer, bufferSize) #endif diff --git a/src/storage/s3/read.c b/src/storage/s3/read.c index 588ecbcd3..bb351a0d3 100644 --- a/src/storage/s3/read.c +++ b/src/storage/s3/read.c @@ -27,7 +27,7 @@ typedef struct StorageReadS3 StorageReadInterface interface; // Interface StorageS3 *storage; // Storage that created this object - HttpClient *httpClient; // HTTP client for requests + HttpResponse *httpResponse; // HTTP response } StorageReadS3; /*********************************************************************************************************************************** @@ -38,15 +38,6 @@ Macros for function logging #define FUNCTION_LOG_STORAGE_READ_S3_FORMAT(value, buffer, bufferSize) \ objToLog(value, "StorageReadS3", buffer, bufferSize) -/*********************************************************************************************************************************** -Mark HTTP client as done so it can be reused -***********************************************************************************************************************************/ -OBJECT_DEFINE_FREE_RESOURCE_BEGIN(STORAGE_READ_S3, LOG, logLevelTrace) -{ - httpClientDone(this->httpClient); -} -OBJECT_DEFINE_FREE_RESOURCE_END(LOG); - /*********************************************************************************************************************************** Open the file ***********************************************************************************************************************************/ @@ -60,16 +51,20 @@ storageReadS3Open(THIS_VOID) FUNCTION_LOG_END(); ASSERT(this != NULL); - ASSERT(this->httpClient == NULL); + ASSERT(this->httpResponse == NULL); bool result = false; // Request the file - this->httpClient = storageS3Request(this->storage, HTTP_VERB_GET_STR, this->interface.name, NULL, NULL, false, true).httpClient; - - if (httpClientResponseCodeOk(this->httpClient)) + MEM_CONTEXT_BEGIN(this->memContext) + { + this->httpResponse = storageS3RequestP( + this->storage, HTTP_VERB_GET_STR, this->interface.name, .allowMissing = true, .contentIo = true); + } + MEM_CONTEXT_END(); + + if (httpResponseCodeOk(this->httpResponse)) { - memContextCallbackSet(this->memContext, storageReadS3FreeResource, this); result = true; } // Else error unless ignore missing @@ -93,33 +88,11 @@ storageReadS3(THIS_VOID, Buffer *buffer, bool block) FUNCTION_LOG_PARAM(BOOL, block); FUNCTION_LOG_END(); - ASSERT(this != NULL && this->httpClient != NULL); - ASSERT(httpClientIoRead(this->httpClient) != NULL); + ASSERT(this != NULL && this->httpResponse != NULL); + ASSERT(httpResponseIoRead(this->httpResponse) != NULL); ASSERT(buffer != NULL && !bufFull(buffer)); - FUNCTION_LOG_RETURN(SIZE, ioRead(httpClientIoRead(this->httpClient), buffer)); -} - -/*********************************************************************************************************************************** -Close the file -***********************************************************************************************************************************/ -static void -storageReadS3Close(THIS_VOID) -{ - THIS(StorageReadS3); - - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(STORAGE_READ_S3, this); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(this->httpClient != NULL); - - memContextCallbackClear(this->memContext); - storageReadS3FreeResource(this); - this->httpClient = NULL; - - FUNCTION_LOG_RETURN_VOID(); + FUNCTION_LOG_RETURN(SIZE, ioRead(httpResponseIoRead(this->httpResponse), buffer)); } /*********************************************************************************************************************************** @@ -134,10 +107,10 @@ storageReadS3Eof(THIS_VOID) FUNCTION_TEST_PARAM(STORAGE_READ_S3, this); FUNCTION_TEST_END(); - ASSERT(this != NULL && this->httpClient != NULL); - ASSERT(httpClientIoRead(this->httpClient) != NULL); + ASSERT(this != NULL && this->httpResponse != NULL); + ASSERT(httpResponseIoRead(this->httpResponse) != NULL); - FUNCTION_TEST_RETURN(ioReadEof(httpClientIoRead(this->httpClient))); + FUNCTION_TEST_RETURN(ioReadEof(httpResponseIoRead(this->httpResponse))); } /**********************************************************************************************************************************/ @@ -172,7 +145,6 @@ storageReadS3New(StorageS3 *storage, const String *name, bool ignoreMissing) .ioInterface = (IoReadInterface) { - .close = storageReadS3Close, .eof = storageReadS3Eof, .open = storageReadS3Open, .read = storageReadS3, diff --git a/src/storage/s3/storage.c b/src/storage/s3/storage.c index 033d75379..459b284f2 100644 --- a/src/storage/s3/storage.c +++ b/src/storage/s3/storage.c @@ -8,7 +8,7 @@ S3 Storage #include "common/crypto/hash.h" #include "common/encode.h" #include "common/debug.h" -#include "common/io/http/cache.h" +#include "common/io/http/client.h" #include "common/io/http/common.h" #include "common/log.h" #include "common/memContext.h" @@ -86,7 +86,7 @@ struct StorageS3 { STORAGE_COMMON_MEMBER; MemContext *memContext; - HttpClientCache *httpClientCache; // HTTP client cache to service requests + HttpClient *httpClient; // HTTP client to service requests StringList *headerRedactList; // List of headers to redact from logging const String *bucket; // Bucket to store data in @@ -234,77 +234,99 @@ storageS3Auth( /*********************************************************************************************************************************** Process S3 request ***********************************************************************************************************************************/ -StorageS3RequestResult -storageS3Request( - StorageS3 *this, const String *verb, const String *uri, const HttpQuery *query, const Buffer *body, bool returnContent, - bool allowMissing) +HttpRequest * +storageS3RequestAsync(StorageS3 *this, const String *verb, const String *uri, StorageS3RequestAsyncParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STORAGE_S3, this); FUNCTION_LOG_PARAM(STRING, verb); FUNCTION_LOG_PARAM(STRING, uri); - FUNCTION_LOG_PARAM(HTTP_QUERY, query); - FUNCTION_LOG_PARAM(BUFFER, body); - FUNCTION_LOG_PARAM(BOOL, returnContent); - FUNCTION_LOG_PARAM(BOOL, allowMissing); + FUNCTION_LOG_PARAM(HTTP_QUERY, param.query); + FUNCTION_LOG_PARAM(BUFFER, param.content); FUNCTION_LOG_END(); ASSERT(this != NULL); ASSERT(verb != NULL); ASSERT(uri != NULL); - StorageS3RequestResult result = {0}; + HttpRequest *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + HttpHeader *requestHeader = httpHeaderNew(this->headerRedactList); + + // Set content length + httpHeaderAdd( + requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR, + param.content == NULL || bufUsed(param.content) == 0 ? ZERO_STR : strNewFmt("%zu", bufUsed(param.content))); + + // Calculate content-md5 header if there is content + if (param.content != NULL) + { + char md5Hash[HASH_TYPE_MD5_SIZE_HEX]; + encodeToStr(encodeBase64, bufPtr(cryptoHashOne(HASH_TYPE_MD5_STR, param.content)), HASH_TYPE_M5_SIZE, md5Hash); + httpHeaderAdd(requestHeader, HTTP_HEADER_CONTENT_MD5_STR, STR(md5Hash)); + } + + // When using path-style URIs the bucket name needs to be prepended + if (this->uriStyle == storageS3UriStylePath) + uri = strNewFmt("/%s%s", strPtr(this->bucket), strPtr(uri)); + + // Generate authorization header + storageS3Auth( + this, verb, httpUriEncode(uri, true), param.query, storageS3DateTime(time(NULL)), requestHeader, + param.content == NULL || bufUsed(param.content) == 0 ? + HASH_TYPE_SHA256_ZERO_STR : bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR, param.content))); + + // Send request + MEM_CONTEXT_PRIOR_BEGIN() + { + result = httpRequestNewP( + this->httpClient, verb, uri, .query = param.query, .header = requestHeader, .content = param.content); + } + MEM_CONTEXT_END(); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(HTTP_REQUEST, result); +} + +HttpResponse * +storageS3Response(HttpRequest *request, StorageS3ResponseParam param) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(HTTP_REQUEST, request); + FUNCTION_LOG_PARAM(BOOL, param.allowMissing); + FUNCTION_LOG_PARAM(BOOL, param.contentIo); + FUNCTION_LOG_END(); + + ASSERT(request != NULL); + + HttpResponse *result = NULL; unsigned int retryRemaining = 2; bool done; - // When using path-style URIs the bucket name needs to be prepended - if (this->uriStyle == storageS3UriStylePath) - uri = strNewFmt("/%s%s", strPtr(this->bucket), strPtr(uri)); - do { done = true; MEM_CONTEXT_TEMP_BEGIN() { - // Create header list and add content length - HttpHeader *requestHeader = httpHeaderNew(this->headerRedactList); - - // Set content length - httpHeaderAdd( - requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR, - body == NULL || bufUsed(body) == 0 ? ZERO_STR : strNewFmt("%zu", bufUsed(body))); - - // Calculate content-md5 header if there is content - if (body != NULL) - { - char md5Hash[HASH_TYPE_MD5_SIZE_HEX]; - encodeToStr(encodeBase64, bufPtr(cryptoHashOne(HASH_TYPE_MD5_STR, body)), HASH_TYPE_M5_SIZE, md5Hash); - httpHeaderAdd(requestHeader, HTTP_HEADER_CONTENT_MD5_STR, STR(md5Hash)); - } - - // Generate authorization header - storageS3Auth( - this, verb, httpUriEncode(uri, true), query, storageS3DateTime(time(NULL)), requestHeader, - body == NULL || bufUsed(body) == 0 ? HASH_TYPE_SHA256_ZERO_STR : bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR, body))); - - // Get an HTTP client - HttpClient *httpClient = httpClientCacheGet(this->httpClientCache); - // Process request - Buffer *response = httpClientRequest(httpClient, verb, uri, query, requestHeader, body, returnContent); + result = httpRequest(request, !param.contentIo); // Error if the request was not successful - if (!httpClientResponseCodeOk(httpClient) && - (!allowMissing || httpClientResponseCode(httpClient) != HTTP_RESPONSE_CODE_NOT_FOUND)) + if (!httpResponseCodeOk(result) && (!param.allowMissing || httpResponseCode(result) != HTTP_RESPONSE_CODE_NOT_FOUND)) { // If there are retries remaining and a response parse it as XML to extract the S3 error code - if (response != NULL && retryRemaining > 0) + const Buffer *content = httpResponseContent(result); + + if (bufUsed(content) > 0 && retryRemaining > 0) { // Attempt to parse the XML and extract the S3 error code TRY_BEGIN() { - XmlNode *error = xmlDocumentRoot(xmlDocumentNewBuf(response)); + XmlNode *error = xmlDocumentRoot(xmlDocumentNewBuf(content)); const String *errorCode = xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_CODE_STR, true)); if (strEq(errorCode, S3_ERROR_REQUEST_TIME_TOO_SKEWED_STR)) @@ -324,75 +346,38 @@ storageS3Request( TRY_END(); } - // If not done then retry instead of reporting the error + // If done throw the error if (done) - { - // General error message - String *error = strNewFmt( - "S3 request failed with %u: %s", httpClientResponseCode(httpClient), - strPtr(httpClientResponseMessage(httpClient))); - - // Output uri/query - strCatZ(error, "\n*** URI/Query ***:"); - - strCatFmt(error, "\n%s", strPtr(httpUriEncode(uri, true))); - - if (query != NULL) - strCatFmt(error, "?%s", strPtr(httpQueryRender(query))); - - // Output request headers - const StringList *requestHeaderList = httpHeaderList(requestHeader); - - strCatZ(error, "\n*** Request Headers ***:"); - - for (unsigned int requestHeaderIdx = 0; requestHeaderIdx < strLstSize(requestHeaderList); requestHeaderIdx++) - { - const String *key = strLstGet(requestHeaderList, requestHeaderIdx); - - strCatFmt( - error, "\n%s: %s", strPtr(key), - httpHeaderRedact(requestHeader, key) || strEq(key, S3_HEADER_DATE_STR) ? - "" : strPtr(httpHeaderGet(requestHeader, key))); - } - - // Output response headers - const HttpHeader *responseHeader = httpClientResponseHeader(httpClient); - const StringList *responseHeaderList = httpHeaderList(responseHeader); - - if (strLstSize(responseHeaderList) > 0) - { - strCatZ(error, "\n*** Response Headers ***:"); - - for (unsigned int responseHeaderIdx = 0; responseHeaderIdx < strLstSize(responseHeaderList); - responseHeaderIdx++) - { - const String *key = strLstGet(responseHeaderList, responseHeaderIdx); - strCatFmt(error, "\n%s: %s", strPtr(key), strPtr(httpHeaderGet(responseHeader, key))); - } - } - - // If there was content then output it - if (response!= NULL) - strCatFmt(error, "\n*** Response Content ***:\n%s", strPtr(strNewBuf(response))); - - THROW(ProtocolError, strPtr(error)); - } + httpRequestError(request, result); } else - { - // On success move the buffer to the prior context - result.httpClient = httpClient; - result.responseHeader = httpHeaderMove( - httpHeaderDup(httpClientResponseHeader(httpClient), NULL), memContextPrior()); - result.response = bufMove(response, memContextPrior()); - } - + httpResponseMove(result, memContextPrior()); } MEM_CONTEXT_TEMP_END(); } while (!done); - FUNCTION_LOG_RETURN(STORAGE_S3_REQUEST_RESULT, result); + FUNCTION_LOG_RETURN(HTTP_RESPONSE, result); +} + +HttpResponse * +storageS3Request(StorageS3 *this, const String *verb, const String *uri, StorageS3RequestParam param) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(STORAGE_S3, this); + FUNCTION_LOG_PARAM(STRING, verb); + FUNCTION_LOG_PARAM(STRING, uri); + FUNCTION_LOG_PARAM(HTTP_QUERY, param.query); + FUNCTION_LOG_PARAM(BUFFER, param.content); + FUNCTION_LOG_PARAM(BOOL, param.allowMissing); + FUNCTION_LOG_PARAM(BOOL, param.contentIo); + FUNCTION_LOG_END(); + + FUNCTION_LOG_RETURN( + HTTP_RESPONSE, + storageS3ResponseP( + storageS3RequestAsyncP(this, verb, uri, .query = param.query, .content = param.content), + .allowMissing = param.allowMissing, .contentIo = param.contentIo)); } /*********************************************************************************************************************************** @@ -469,8 +454,7 @@ storageS3ListInternal( httpQueryAdd(query, S3_QUERY_PREFIX_STR, queryPrefix); XmlNode *xmlRoot = xmlDocumentRoot( - xmlDocumentNewBuf( - storageS3Request(this, HTTP_VERB_GET_STR, FSLASH_STR, query, NULL, true, false).response)); + xmlDocumentNewBuf(httpResponseContent(storageS3RequestP(this, HTTP_VERB_GET_STR, FSLASH_STR, query)))); // Get subpath list XmlNodeList *subPathList = xmlNodeChildList(xmlRoot, S3_XML_TAG_COMMON_PREFIXES_STR); @@ -539,17 +523,19 @@ storageS3Info(THIS_VOID, const String *file, StorageInfoLevel level, StorageInte ASSERT(file != NULL); // Attempt to get file info - StorageS3RequestResult httpResult = storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, true, true); + HttpResponse *httpResponse = storageS3RequestP(this, HTTP_VERB_HEAD_STR, file, .allowMissing = true); // Does the file exist? - StorageInfo result = {.level = level, .exists = httpClientResponseCodeOk(httpResult.httpClient)}; + StorageInfo result = {.level = level, .exists = httpResponseCodeOk(httpResponse)}; // Add basic level info if requested and the file exists if (result.level >= storageInfoLevelBasic && result.exists) { + const HttpHeader *httpHeader = httpResponseHeader(httpResponse); + result.type = storageTypeFile; - result.size = cvtZToUInt64(strPtr(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_CONTENT_LENGTH_STR))); - result.timeModified = httpDateToTime(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_LAST_MODIFIED_STR)); + result.size = cvtZToUInt64(strPtr(httpHeaderGet(httpHeader, HTTP_HEADER_CONTENT_LENGTH_STR))); + result.timeModified = httpDateToTime(httpHeaderGet(httpHeader, HTTP_HEADER_LAST_MODIFIED_STR)); } FUNCTION_LOG_RETURN(STORAGE_INFO, result); @@ -708,12 +694,13 @@ storageS3PathRemoveInternal(StorageS3 *this, XmlDocument *request) ASSERT(this != NULL); ASSERT(request != NULL); - Buffer *response = storageS3Request( - this, HTTP_VERB_POST_STR, FSLASH_STR, httpQueryAdd(httpQueryNew(), S3_QUERY_DELETE_STR, EMPTY_STR), - xmlDocumentBuf(request), true, false).response; + const Buffer *response = httpResponseContent( + storageS3RequestP( + this, HTTP_VERB_POST_STR, FSLASH_STR, .query = httpQueryAdd(httpQueryNew(), S3_QUERY_DELETE_STR, EMPTY_STR), + .content = xmlDocumentBuf(request))); // Nothing is returned when there are no errors - if (response != NULL) + if (bufSize(response) > 0) { XmlNodeList *errorList = xmlNodeChildList(xmlDocumentRoot(xmlDocumentNewBuf(response)), S3_XML_TAG_ERROR_STR); @@ -828,7 +815,7 @@ storageS3Remove(THIS_VOID, const String *file, StorageInterfaceRemoveParam param ASSERT(file != NULL); ASSERT(!param.errorOnMissing); - storageS3Request(this, HTTP_VERB_DELETE_STR, file, NULL, NULL, true, false); + storageS3RequestP(this, HTTP_VERB_DELETE_STR, file); FUNCTION_LOG_RETURN_VOID(); } @@ -904,13 +891,14 @@ storageS3New( .signingKeyDate = YYYYMMDD_STR, }; - // Create the HTTP client cache used to service requests - driver->httpClientCache = httpClientCacheNew( + // Create the HTTP client used to service requests + driver->httpClient = httpClientNew( host == NULL ? driver->bucketEndpoint : host, driver->port, timeout, verifyPeer, caFile, caPath); // Create list of redacted headers driver->headerRedactList = strLstNew(); strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR); + strLstAdd(driver->headerRedactList, S3_HEADER_DATE_STR); this = storageNew( STORAGE_S3_TYPE_STR, path, 0, 0, write, pathExpressionFunction, driver, driver->interface); diff --git a/src/storage/s3/storage.intern.h b/src/storage/s3/storage.intern.h index 26eba4fdd..51ff48591 100644 --- a/src/storage/s3/storage.intern.h +++ b/src/storage/s3/storage.intern.h @@ -9,27 +9,52 @@ Object type ***********************************************************************************************************************************/ typedef struct StorageS3 StorageS3; -#include "common/io/http/client.h" +#include "common/io/http/request.h" #include "storage/s3/storage.h" /*********************************************************************************************************************************** -Perform an S3 Request +Functions ***********************************************************************************************************************************/ -#define FUNCTION_LOG_STORAGE_S3_REQUEST_RESULT_TYPE \ - StorageS3RequestResult -#define FUNCTION_LOG_STORAGE_S3_REQUEST_RESULT_FORMAT(value, buffer, bufferSize) \ - objToLog(&value, "StorageS3RequestResult", buffer, bufferSize) - -typedef struct StorageS3RequestResult +// Perform async request +typedef struct StorageS3RequestAsyncParam { - HttpClient *httpClient; - HttpHeader *responseHeader; - Buffer *response; -} StorageS3RequestResult; + VAR_PARAM_HEADER; + const HttpQuery *query; + const Buffer *content; +} StorageS3RequestAsyncParam; -StorageS3RequestResult storageS3Request( - StorageS3 *this, const String *verb, const String *uri, const HttpQuery *query, const Buffer *body, bool returnContent, - bool allowMissing); +#define storageS3RequestAsyncP(this, verb, uri, ...) \ + storageS3RequestAsync(this, verb, uri, (StorageS3RequestAsyncParam){VAR_PARAM_INIT, __VA_ARGS__}) + +HttpRequest *storageS3RequestAsync(StorageS3 *this, const String *verb, const String *uri, StorageS3RequestAsyncParam param); + +// Get async response +typedef struct StorageS3ResponseParam +{ + VAR_PARAM_HEADER; + bool allowMissing; + bool contentIo; +} StorageS3ResponseParam; + +#define storageS3ResponseP(request, ...) \ + storageS3Response(request, (StorageS3ResponseParam){VAR_PARAM_INIT, __VA_ARGS__}) + +HttpResponse *storageS3Response(HttpRequest *request, StorageS3ResponseParam param); + +// Perform sync request +typedef struct StorageS3RequestParam +{ + VAR_PARAM_HEADER; + const HttpQuery *query; + const Buffer *content; + bool allowMissing; + bool contentIo; +} StorageS3RequestParam; + +#define storageS3RequestP(this, verb, uri, ...) \ + storageS3Request(this, verb, uri, (StorageS3RequestParam){VAR_PARAM_INIT, __VA_ARGS__}) + +HttpResponse *storageS3Request(StorageS3 *this, const String *verb, const String *uri, StorageS3RequestParam param); /*********************************************************************************************************************************** Macros for function logging diff --git a/src/storage/s3/write.c b/src/storage/s3/write.c index b6a173c0f..d33a3ddb9 100644 --- a/src/storage/s3/write.c +++ b/src/storage/s3/write.c @@ -37,6 +37,7 @@ typedef struct StorageWriteS3 StorageWriteInterface interface; // Interface StorageS3 *storage; // Storage that created this object + HttpRequest *request; // Async request size_t partSize; Buffer *partBuffer; const String *uploadId; @@ -86,21 +87,48 @@ storageWriteS3Part(StorageWriteS3 *this) FUNCTION_LOG_PARAM(STORAGE_WRITE_S3, this); FUNCTION_LOG_END(); + ASSERT(this != NULL); + + // If there is an outstanding async request then wait for the response and store the part id + if (this->request != NULL) + { + strLstAdd( + this->uploadPartList, httpHeaderGet(httpResponseHeader(storageS3ResponseP(this->request)), HTTP_HEADER_ETAG_STR)); + ASSERT(strLstGet(this->uploadPartList, strLstSize(this->uploadPartList) - 1) != NULL); + + httpRequestFree(this->request); + this->request = NULL; + } + + FUNCTION_LOG_RETURN_VOID(); +} + +static void +storageWriteS3PartAsync(StorageWriteS3 *this) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(STORAGE_WRITE_S3, this); + FUNCTION_LOG_END(); + ASSERT(this != NULL); ASSERT(this->partBuffer != NULL); ASSERT(bufSize(this->partBuffer) > 0); MEM_CONTEXT_TEMP_BEGIN() { + // Complete prior async request, if any + storageWriteS3Part(this); + // Get the upload id if we have not already if (this->uploadId == NULL) { // Initiate mult-part upload XmlNode *xmlRoot = xmlDocumentRoot( xmlDocumentNewBuf( - storageS3Request( - this->storage, HTTP_VERB_POST_STR, this->interface.name, - httpQueryAdd(httpQueryNew(), S3_QUERY_UPLOADS_STR, EMPTY_STR), NULL, true, false).response)); + httpResponseContent( + storageS3RequestP( + this->storage, HTTP_VERB_POST_STR, this->interface.name, + .query = httpQueryAdd(httpQueryNew(), S3_QUERY_UPLOADS_STR, EMPTY_STR))))); // Store the upload id MEM_CONTEXT_BEGIN(this->memContext) @@ -111,19 +139,17 @@ storageWriteS3Part(StorageWriteS3 *this) MEM_CONTEXT_END(); } - // Upload the part and add etag to part list + // Upload the part async HttpQuery *query = httpQueryNew(); httpQueryAdd(query, S3_QUERY_UPLOAD_ID_STR, this->uploadId); httpQueryAdd(query, S3_QUERY_PART_NUMBER_STR, strNewFmt("%u", strLstSize(this->uploadPartList) + 1)); - strLstAdd( - this->uploadPartList, - httpHeaderGet( - storageS3Request( - this->storage, HTTP_VERB_PUT_STR, this->interface.name, query, this->partBuffer, true, false).responseHeader, - HTTP_HEADER_ETAG_STR)); - - ASSERT(strLstGet(this->uploadPartList, strLstSize(this->uploadPartList) - 1) != NULL); + MEM_CONTEXT_BEGIN(this->memContext) + { + this->request = storageS3RequestAsyncP( + this->storage, HTTP_VERB_PUT_STR, this->interface.name, .query = query, .content = this->partBuffer); + } + MEM_CONTEXT_END(); } MEM_CONTEXT_TEMP_END(); @@ -145,6 +171,7 @@ storageWriteS3(THIS_VOID, const Buffer *buffer) ASSERT(this != NULL); ASSERT(this->partBuffer != NULL); + ASSERT(buffer != NULL); size_t bytesTotal = 0; @@ -160,7 +187,7 @@ storageWriteS3(THIS_VOID, const Buffer *buffer) // If the part buffer is full then write it if (bufRemains(this->partBuffer) == 0) { - storageWriteS3Part(this); + storageWriteS3PartAsync(this); bufUsedZero(this->partBuffer); } } @@ -193,7 +220,10 @@ storageWriteS3Close(THIS_VOID) { // If there is anything left in the part buffer then write it if (bufUsed(this->partBuffer) > 0) - storageWriteS3Part(this); + storageWriteS3PartAsync(this); + + // Complete prior async request, if any + storageWriteS3Part(this); // Generate the xml part list XmlDocument *partList = xmlDocumentNew(S3_XML_TAG_COMPLETE_MULTIPART_UPLOAD_STR); @@ -206,16 +236,14 @@ storageWriteS3Close(THIS_VOID) } // Finalize the multi-part upload - storageS3Request( + storageS3RequestP( this->storage, HTTP_VERB_POST_STR, this->interface.name, - httpQueryAdd(httpQueryNew(), S3_QUERY_UPLOAD_ID_STR, this->uploadId), xmlDocumentBuf(partList), true, false); + .query = httpQueryAdd(httpQueryNew(), S3_QUERY_UPLOAD_ID_STR, this->uploadId), + .content = xmlDocumentBuf(partList)); } // Else upload all the data in a single put else - { - storageS3Request( - this->storage, HTTP_VERB_PUT_STR, this->interface.name, NULL, this->partBuffer, true, false); - } + storageS3RequestP(this->storage, HTTP_VERB_PUT_STR, this->interface.name, .content = this->partBuffer); bufFree(this->partBuffer); this->partBuffer = NULL; diff --git a/test/define.yaml b/test/define.yaml index e9a7c0fce..9cf153b5c 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -250,14 +250,16 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: io-http - total: 6 + total: 5 coverage: - common/io/http/cache: full common/io/http/client: full common/io/http/common: full common/io/http/header: full common/io/http/query: full + common/io/http/request: full + common/io/http/response: full + common/io/http/session: full # ---------------------------------------------------------------------------------------------------------------------------- - name: compress diff --git a/test/expect/mock-stanza-002.log b/test/expect/mock-stanza-002.log index 0b295c979..709e59bdb 100644 --- a/test/expect/mock-stanza-002.log +++ b/test/expect/mock-stanza-002.log @@ -6,6 +6,9 @@ stanza-create db - fail on missing control file (backup host) ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db P00 ERROR: [055]: raised from remote-0 protocol on 'db-primary': unable to open missing file '[TEST_PATH]/db-primary/db/base/global/pg_control' for read +P00 DETAIL: socket statistics:[SOCKET-STATISTICS] +P00 DETAIL: tls statistics:[TLS-STATISTICS] +P00 INFO: http statistics:[HTTP-STATISTICS] P00 INFO: stanza-create command end: aborted with exception [055] stanza-upgrade db - fail on stanza not initialized since archive.info is missing (backup host) diff --git a/test/src/common/harnessTest.h b/test/src/common/harnessTest.h index 8aba54ce5..d5902d7d3 100644 --- a/test/src/common/harnessTest.h +++ b/test/src/common/harnessTest.h @@ -108,7 +108,7 @@ Test that an expected error is actually thrown and error when it isn't \ if (strcmp(errorMessage(), errorMessageExpected) != 0 || errorType() != &errorTypeExpected) \ THROW_FMT( \ - TestError, "EXPECTED %s: %s\n\n BUT GOT %s: %s\n\nTHROWN AT:\n%s", errorTypeName(&errorTypeExpected), \ + TestError, "EXPECTED %s: %s\n\nBUT GOT %s: %s\n\nTHROWN AT:\n%s", errorTypeName(&errorTypeExpected), \ errorMessageExpected, errorName(), errorMessage(), errorStackTrace()); \ } \ TRY_END(); \ diff --git a/test/src/module/common/ioHttpTest.c b/test/src/module/common/ioHttpTest.c index 20321b053..a521dd10c 100644 --- a/test/src/module/common/ioHttpTest.c +++ b/test/src/module/common/ioHttpTest.c @@ -131,7 +131,7 @@ testRun(void) HttpClient *client = NULL; // Reset statistics - httpClientStatLocal = (HttpClientStat){0}; + httpClientStat = (HttpClientStat){0}; TEST_RESULT_STR(httpClientStatStr(), NULL, "no stats yet"); @@ -140,7 +140,7 @@ testRun(void) "new client"); TEST_ERROR_FMT( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), HostConnectError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), HostConnectError, "unable to connect to 'localhost:%u': [111] Connection refused", hrnTlsServerPort()); HARNESS_FORK_BEGIN() @@ -180,7 +180,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FileReadError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FileReadError, "unexpected eof while reading line"); // ----------------------------------------------------------------------------------------------------------------- @@ -194,7 +194,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FormatError, "HTTP response status 'HTTP/1.0 200 OK' should be CR-terminated"); // ----------------------------------------------------------------------------------------------------------------- @@ -208,7 +208,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FormatError, "HTTP response 'HTTP/1.0 200' has invalid length"); // ----------------------------------------------------------------------------------------------------------------- @@ -222,7 +222,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FormatError, "HTTP version of response 'HTTP/1.0 200 OK' must be HTTP/1.1"); // ----------------------------------------------------------------------------------------------------------------- @@ -236,7 +236,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FormatError, "response status '200OK' must have a space after the status code"); // ----------------------------------------------------------------------------------------------------------------- @@ -250,7 +250,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FileReadError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FileReadError, "unexpected eof while reading line"); // ----------------------------------------------------------------------------------------------------------------- @@ -264,7 +264,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FormatError, "header 'header-value' missing colon"); // ----------------------------------------------------------------------------------------------------------------- @@ -278,7 +278,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FormatError, "only 'chunked' is supported for 'transfer-encoding' header"); // ----------------------------------------------------------------------------------------------------------------- @@ -292,7 +292,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), FormatError, "'transfer-encoding' and 'content-length' headers are both set"); // ----------------------------------------------------------------------------------------------------------------- @@ -306,7 +306,7 @@ testRun(void) hrnTlsServerClose(); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), ServiceError, + httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), ServiceError, "[503] Slow Down"); // ----------------------------------------------------------------------------------------------------------------- @@ -321,7 +321,7 @@ testRun(void) hrnTlsServerAccept(); hrnTlsServerExpectZ("GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\nhost:myhost.com\r\n\r\n"); - hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\nkey1:0\r\n key2 : value2\r\nConnection:ack\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\nkey1:0\r\n key2 : value2\r\nConnection:ack\r\ncontent-length:0\r\n\r\n"); HttpHeader *headerRequest = httpHeaderNew(NULL); httpHeaderAdd(headerRequest, strNew("host"), strNew("myhost.com")); @@ -332,14 +332,38 @@ testRun(void) client->timeout = 5000; - TEST_RESULT_VOID( - httpClientRequest(client, strNew("GET"), strNew("/"), query, headerRequest, NULL, false), "request"); - TEST_RESULT_UINT(httpClientResponseCode(client), 200, "check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", "check response message"); - TEST_RESULT_UINT(httpClientEof(client), true, "io is eof"); + HttpRequest *request = NULL; + HttpResponse *response = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + TEST_ASSIGN( + request, httpRequestNewP(client, strNew("GET"), strNew("/"), .query = query, .header = headerRequest), + "request"); + TEST_ASSIGN(response, httpRequest(request, false), "request"); + + TEST_RESULT_VOID(httpRequestMove(request, memContextPrior()), "move request"); + TEST_RESULT_VOID(httpResponseMove(response, memContextPrior()), "move response"); + } + MEM_CONTEXT_TEMP_END(); + + TEST_RESULT_STR_Z(httpRequestVerb(request), "GET", "check request verb"); + TEST_RESULT_STR_Z(httpRequestUri(request), "/", "check request uri"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'ack', key1: '0', key2: 'value2'}", - "check response headers"); + httpQueryRender(httpRequestQuery(request)), "name=%2Fpath%2FA%20Z.txt&type=test", "check request query"); + TEST_RESULT_PTR_NE(httpRequestHeader(request), NULL, "check request headers"); + + TEST_RESULT_UINT(httpResponseCode(response), 200, "check response code"); + TEST_RESULT_BOOL(httpResponseCodeOk(response), true, "check response code ok"); + TEST_RESULT_STR_Z(httpResponseReason(response), "OK", "check response message"); + TEST_RESULT_UINT(httpResponseEof(response), true, "io is eof"); + TEST_RESULT_STR_Z( + httpHeaderToLog(httpResponseHeader(response)), + "{connection: 'ack', content-length: '0', key1: '0', key2: 'value2'}", "check response headers"); + TEST_RESULT_UINT(bufSize(httpResponseContent(response)), 0, "content is empty"); + + TEST_RESULT_VOID(httpResponseFree(response), "free response"); + TEST_RESULT_VOID(httpRequestFree(request), "free request"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("head request with content-length but no content"); @@ -347,14 +371,13 @@ testRun(void) hrnTlsServerExpectZ("HEAD / HTTP/1.1\r\n\r\n"); hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\ncontent-length:380\r\n\r\n"); - TEST_RESULT_VOID( - httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), "request"); - TEST_RESULT_UINT(httpClientResponseCode(client), 200, "check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", "check response message"); - TEST_RESULT_BOOL(httpClientEof(client), true, "io is eof"); - TEST_RESULT_BOOL(httpClientBusy(client), false, "client is not busy"); + TEST_ASSIGN(response, httpRequest(httpRequestNewP(client, strNew("HEAD"), strNew("/")), true), "request"); + TEST_RESULT_UINT(httpResponseCode(response), 200, "check response code"); + TEST_RESULT_STR_Z(httpResponseReason(response), "OK", "check response message"); + TEST_RESULT_BOOL(httpResponseEof(response), true, "io is eof"); + TEST_RESULT_PTR(response->session, NULL, "session is not busy"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '380'}", "check response headers"); + httpHeaderToLog(httpResponseHeader(response)), "{content-length: '380'}", "check response headers"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("head request with transfer encoding but no content"); @@ -362,14 +385,13 @@ testRun(void) hrnTlsServerExpectZ("HEAD / HTTP/1.1\r\n\r\n"); hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"); - TEST_RESULT_VOID( - httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), "request"); - TEST_RESULT_UINT(httpClientResponseCode(client), 200, "check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", "check response message"); - TEST_RESULT_BOOL(httpClientEof(client), true, "io is eof"); - TEST_RESULT_BOOL(httpClientBusy(client), false, "client is not busy"); + TEST_ASSIGN(response, httpRequest(httpRequestNewP(client, strNew("HEAD"), strNew("/")), true), "request"); + TEST_RESULT_UINT(httpResponseCode(response), 200, "check response code"); + TEST_RESULT_STR_Z(httpResponseReason(response), "OK", "check response message"); + TEST_RESULT_BOOL(httpResponseEof(response), true, "io is eof"); + TEST_RESULT_PTR(response->session, NULL, "session is not busy"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{transfer-encoding: 'chunked'}", "check response headers"); + httpHeaderToLog(httpResponseHeader(response)), "{transfer-encoding: 'chunked'}", "check response headers"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("head request with connection close but no content"); @@ -379,14 +401,13 @@ testRun(void) hrnTlsServerClose(); - TEST_RESULT_VOID( - httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), "request"); - TEST_RESULT_UINT(httpClientResponseCode(client), 200, "check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", "check response message"); - TEST_RESULT_BOOL(httpClientEof(client), true, "io is eof"); - TEST_RESULT_BOOL(httpClientBusy(client), false, "client is not busy"); + TEST_ASSIGN(response, httpRequest(httpRequestNewP(client, strNew("HEAD"), strNew("/")), true), "request"); + TEST_RESULT_UINT(httpResponseCode(response), 200, "check response code"); + TEST_RESULT_STR_Z(httpResponseReason(response), "OK", "check response message"); + TEST_RESULT_BOOL(httpResponseEof(response), true, "io is eof"); + TEST_RESULT_PTR(response->session, NULL, "session is not busy"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", "check response headers"); + httpHeaderToLog(httpResponseHeader(response)), "{connection: 'close'}", "check response headers"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("error with content (with a few slow down errors)"); @@ -406,28 +427,59 @@ testRun(void) hrnTlsServerAccept(); hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); - hrnTlsServerReplyZ("HTTP/1.1 404 Not Found\r\ncontent-length:0\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 404 Not Found\r\n\r\n"); - TEST_RESULT_VOID(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "request"); - TEST_RESULT_UINT(httpClientResponseCode(client), 404, "check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "Not Found", "check response message"); + TEST_ASSIGN(request, httpRequestNewP(client, strNew("GET"), strNew("/")), "request"); + TEST_ASSIGN(response, httpRequest(request, false), "response"); + TEST_RESULT_UINT(httpResponseCode(response), 404, "check response code"); + TEST_RESULT_BOOL(httpResponseCodeOk(response), false, "check response code error"); + TEST_RESULT_STR_Z(httpResponseReason(response), "Not Found", "check response message"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '0'}", "check response headers"); + httpHeaderToLog(httpResponseHeader(response)), "{}", "check response headers"); + + TEST_ERROR( + httpRequestError(request, response), ProtocolError, + "HTTP request failed with 404 (Not Found):\n" + "*** URI/Query ***:\n" + "/"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("error with content"); - hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerExpectZ("GET /?a=b HTTP/1.1\r\nhdr1:1\r\nhdr2:2\r\n\r\n"); hrnTlsServerReplyZ("HTTP/1.1 403 \r\ncontent-length:7\r\n\r\nCONTENT"); - Buffer *buffer = NULL; + StringList *headerRedact = strLstNew(); + strLstAdd(headerRedact, STRDEF("hdr2")); + headerRequest = httpHeaderNew(headerRedact); + httpHeaderAdd(headerRequest, strNew("hdr1"), strNew("1")); + httpHeaderAdd(headerRequest, strNew("hdr2"), strNew("2")); - TEST_ASSIGN(buffer, httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "request"); - TEST_RESULT_UINT(httpClientResponseCode(client), 403, "check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "", "check empty response message"); + TEST_ASSIGN( + request, + httpRequestNewP( + client, strNew("GET"), strNew("/"), .query = httpQueryAdd(httpQueryNew(), STRDEF("a"), STRDEF("b")), + .header = headerRequest), + "request"); + TEST_ASSIGN(response, httpRequest(request, false), "response"); + TEST_RESULT_UINT(httpResponseCode(response), 403, "check response code"); + TEST_RESULT_STR_Z(httpResponseReason(response), "", "check empty response message"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '7'}", "check response headers"); - TEST_RESULT_STR_Z(strNewBuf(buffer), "CONTENT", "check response"); + httpHeaderToLog(httpResponseHeader(response)), "{content-length: '7'}", "check response headers"); + TEST_RESULT_STR_Z(strNewBuf(httpResponseContent(response)), "CONTENT", "check response content"); + + TEST_ERROR( + httpRequestError(request, response), ProtocolError, + "HTTP request failed with 403:\n" + "*** URI/Query ***:\n" + "/?a=b\n" + "*** Request Headers ***:\n" + "hdr1: 1\n" + "hdr2: \n" + "*** Response Headers ***:\n" + "content-length: 7\n" + "*** Response Content ***:\n" + "CONTENT"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("request with content using content-length"); @@ -440,16 +492,17 @@ testRun(void) ioBufferSizeSet(30); TEST_ASSIGN( - buffer, - httpClientRequest( - client, strNew("GET"), strNew("/path/file 1.txt"), NULL, - httpHeaderAdd(httpHeaderNew(NULL), strNew("content-length"), strNew("30")), - BUFSTRDEF("012345678901234567890123456789"), true), + response, + httpRequest( + httpRequestNewP( + client, strNew("GET"), strNew("/path/file 1.txt"), + .header = httpHeaderAdd(httpHeaderNew(NULL), strNew("content-length"), strNew("30")), + .content = BUFSTRDEF("012345678901234567890123456789")), true), "request"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", "check response headers"); - TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901", "check response"); - TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, "call internal read to check eof"); + httpHeaderToLog(httpResponseHeader(response)), "{connection: 'close'}", "check response headers"); + TEST_RESULT_STR_Z(strNewBuf(httpResponseContent(response)), "01234567890123456789012345678901", "check response"); + TEST_RESULT_UINT(httpResponseRead(response, bufNew(1), true), 0, "call internal read to check eof"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("request with eof before content complete with retry"); @@ -466,10 +519,10 @@ testRun(void) hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\ncontent-length:32\r\n\r\n01234567890123456789012345678901"); TEST_ASSIGN( - buffer, httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, NULL, true), + response, httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/path/file 1.txt")), true), "request"); - TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901", "check response"); - TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, "call internal read to check eof"); + TEST_RESULT_STR_Z(strNewBuf(httpResponseContent(response)), "01234567890123456789012345678901", "check response"); + TEST_RESULT_UINT(httpResponseRead(response, bufNew(1), true), 0, "call internal read to check eof"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("request with eof before content complete"); @@ -479,12 +532,12 @@ testRun(void) hrnTlsServerClose(); - buffer = bufNew(32); - - TEST_RESULT_VOID( - httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, NULL, false), "request"); - TEST_RESULT_BOOL(httpClientBusy(client), true, "client is busy"); - TEST_ERROR(ioRead(httpClientIoRead(client), buffer), FileReadError, "unexpected EOF reading HTTP content"); + TEST_ASSIGN( + response, httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/path/file 1.txt")), false), + "request"); + TEST_RESULT_PTR_NE(response->session, NULL, "session is busy"); + TEST_ERROR(ioRead(httpResponseIoRead(response), bufNew(32)), FileReadError, "unexpected EOF reading HTTP content"); + TEST_RESULT_PTR_NE(response->session, NULL, "session is still busy"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("request with chunked content"); @@ -498,23 +551,19 @@ testRun(void) "10\r\n0123456789012345\r\n" "0\r\n\r\n"); - TEST_RESULT_VOID(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "request"); + TEST_ASSIGN(response, httpRequest(httpRequestNewP(client, strNew("GET"), strNew("/")), false), "request"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{transfer-encoding: 'chunked'}", "check response headers"); + httpHeaderToLog(httpResponseHeader(response)), "{transfer-encoding: 'chunked'}", "check response headers"); - buffer = bufNew(35); + Buffer *buffer = bufNew(35); - TEST_RESULT_VOID(ioRead(httpClientIoRead(client), buffer), "read response"); + TEST_RESULT_VOID(ioRead(httpResponseIoRead(response), buffer), "read response"); TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901012", "check response"); // ----------------------------------------------------------------------------------------------------------------- - TEST_TITLE("close connection"); + TEST_TITLE("close connection and end server process"); hrnTlsServerClose(); - - TEST_RESULT_VOID(httpClientFree(client), "free client"); - - // ----------------------------------------------------------------------------------------------------------------- hrnTlsClientEnd(); } HARNESS_FORK_PARENT_END(); @@ -527,31 +576,5 @@ testRun(void) TEST_RESULT_BOOL(httpClientStatStr() != NULL, true, "check"); } - // ***************************************************************************************************************************** - if (testBegin("HttpClientCache")) - { - HttpClientCache *cache = NULL; - HttpClient *client1 = NULL; - HttpClient *client2 = NULL; - - TEST_ASSIGN( - cache, httpClientCacheNew(strNew("localhost"), hrnTlsServerPort(), 5000, true, NULL, NULL), "new HTTP client cache"); - TEST_ASSIGN(client1, httpClientCacheGet(cache), "get HTTP client"); - TEST_RESULT_PTR(client1, *(HttpClient **)lstGet(cache->clientList, 0), " check HTTP client"); - TEST_RESULT_PTR(httpClientCacheGet(cache), *(HttpClient **)lstGet(cache->clientList, 0), " get same HTTP client"); - - // Make client 1 look like it is busy - client1->ioRead = (IoRead *)1; - - TEST_ASSIGN(client2, httpClientCacheGet(cache), "get HTTP client"); - TEST_RESULT_PTR(client2, *(HttpClient **)lstGet(cache->clientList, 1), " check HTTP client"); - TEST_RESULT_BOOL(client1 != client2, true, "clients are not the same"); - - // Set back to NULL so bad things don't happen during free - client1->ioRead = NULL; - - TEST_RESULT_VOID(httpClientCacheFree(cache), "free HTTP client cache"); - } - FUNCTION_HARNESS_RESULT_VOID(); } diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index 4eb303aab..c2b1f6575 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -202,8 +202,10 @@ testRun(void) strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region))); strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s", strPtr(endPoint))); strLstAdd(argList, strNewFmt("--repo1-s3-host=%s", strPtr(host))); +#ifdef TEST_CONTAINER_REQUIRED strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_TEST_CERT); +#endif setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); @@ -229,8 +231,6 @@ testRun(void) strLstAdd(argList, strNewFmt("--repo1-s3-bucket=%s", strPtr(bucket))); strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region))); strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s:999", strPtr(endPoint))); - strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); - strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_TEST_CERT); setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); @@ -257,8 +257,6 @@ testRun(void) strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region))); strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s:999", strPtr(endPoint))); strLstAdd(argList, strNewFmt("--repo1-s3-host=%s:7777", strPtr(host))); - strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); - strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_TEST_CERT); setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); @@ -286,8 +284,6 @@ testRun(void) strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s:999", strPtr(endPoint))); strLstAdd(argList, strNewFmt("--repo1-s3-host=%s:7777", strPtr(host))); strLstAddZ(argList, "--repo1-s3-port=9001"); - strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); - strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_TEST_CERT); setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); @@ -448,7 +444,7 @@ testRun(void) TEST_ERROR( ioReadOpen(storageReadIo(read)), ProtocolError, - "S3 request failed with 303: \n" + "HTTP request failed with 303:\n" "*** URI/Query ***:\n" "/file.txt\n" "*** Request Headers ***:\n" @@ -627,7 +623,7 @@ testRun(void) testResponseP(.code = 344); TEST_ERROR(storageListP(s3, strNew("/")), ProtocolError, - "S3 request failed with 344: \n" + "HTTP request failed with 344:\n" "*** URI/Query ***:\n" "/?delimiter=%2F&list-type=2\n" "*** Request Headers ***:\n" @@ -650,7 +646,7 @@ testRun(void) ""); TEST_ERROR(storageListP(s3, strNew("/")), ProtocolError, - "S3 request failed with 344: \n" + "HTTP request failed with 344:\n" "*** URI/Query ***:\n" "/?delimiter=%2F&list-type=2\n" "*** Request Headers ***:\n" @@ -698,7 +694,7 @@ testRun(void) ""); TEST_ERROR(storageListP(s3, strNew("/")), ProtocolError, - "S3 request failed with 403: Forbidden\n" + "HTTP request failed with 403 (Forbidden):\n" "*** URI/Query ***:\n" "/?delimiter=%2F&list-type=2\n" "*** Request Headers ***:\n"