mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-06-02 22:57:34 +02:00
72252ed2a1
A robust HTTP client with pipelining support and automatic retries. Using a single object to make multiple requests is more efficient because requests are pipelined whenever possible. Requests are automatically retried when the connection has been closed by the server. Any 5xx response is also retried. Only the HTTPS protocol is currently supported.
494 lines
21 KiB
C
494 lines
21 KiB
C
/***********************************************************************************************************************************
|
|
Http Client
|
|
***********************************************************************************************************************************/
|
|
#include "common/assert.h"
|
|
#include "common/debug.h"
|
|
#include "common/io/http/client.h"
|
|
#include "common/io/http/common.h"
|
|
#include "common/io/io.h"
|
|
#include "common/io/read.intern.h"
|
|
#include "common/io/tls/client.h"
|
|
#include "common/log.h"
|
|
#include "common/wait.h"
|
|
|
|
/***********************************************************************************************************************************
|
|
Http constants
|
|
***********************************************************************************************************************************/
|
|
#define HTTP_VERSION "HTTP/1.1"
|
|
STRING_STATIC(HTTP_VERSION_STR, HTTP_VERSION);
|
|
|
|
STRING_EXTERN(HTTP_VERB_GET_STR, HTTP_VERB_GET);
|
|
|
|
#define HTTP_HEADER_CONNECTION "connection"
|
|
STRING_STATIC(HTTP_HEADER_CONNECTION_STR, HTTP_HEADER_CONNECTION);
|
|
STRING_EXTERN(HTTP_HEADER_CONTENT_LENGTH_STR, HTTP_HEADER_CONTENT_LENGTH);
|
|
#define HTTP_HEADER_TRANSFER_ENCODING "transfer-encoding"
|
|
STRING_STATIC(HTTP_HEADER_TRANSFER_ENCODING_STR, HTTP_HEADER_TRANSFER_ENCODING);
|
|
|
|
#define HTTP_VALUE_CONNECTION_CLOSE "close"
|
|
STRING_STATIC(HTTP_VALUE_CONNECTION_CLOSE_STR, HTTP_VALUE_CONNECTION_CLOSE);
|
|
#define HTTP_VALUE_TRANSFER_ENCODING_CHUNKED "chunked"
|
|
STRING_STATIC(HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED);
|
|
|
|
// 5xx errors that should always be retried
|
|
#define HTTP_RESPONSE_CODE_RETRY_CLASS 5
|
|
|
|
/***********************************************************************************************************************************
|
|
Object type
|
|
***********************************************************************************************************************************/
|
|
struct HttpClient
|
|
{
|
|
MemContext *memContext; // Mem context
|
|
TimeMSec timeout; // Request timeout
|
|
|
|
TlsClient *tls; // Tls client
|
|
IoRead *ioRead; // Read io interface
|
|
|
|
unsigned int responseCode; // Response code (e.g. 200, 404)
|
|
String *responseMessage; // Response message e.g. (OK, Not Found)
|
|
HttpHeader *responseHeader; // Response headers
|
|
|
|
bool contentChunked; // Is the reponse content chunked?
|
|
uint64_t contentSize; // Content size (ignored for chunked)
|
|
uint64_t contentRemaining; // Content remaining (per chunk if chunked)
|
|
bool closeOnContentEof; // Will server close after content is sent?
|
|
bool contentEof; // Has all content been read?
|
|
};
|
|
|
|
/***********************************************************************************************************************************
|
|
Read content
|
|
***********************************************************************************************************************************/
|
|
static size_t
|
|
httpClientRead(HttpClient *this, Buffer *buffer)
|
|
{
|
|
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
|
FUNCTION_DEBUG_PARAM(HTTP_CLIENT, this);
|
|
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
|
|
|
|
FUNCTION_TEST_ASSERT(this != NULL);
|
|
FUNCTION_TEST_ASSERT(buffer != NULL);
|
|
FUNCTION_TEST_ASSERT(!bufFull(buffer));
|
|
FUNCTION_DEBUG_END();
|
|
|
|
// Read if EOF has not been reached
|
|
size_t actualBytes = 0;
|
|
|
|
if (!this->contentEof)
|
|
{
|
|
do
|
|
{
|
|
// If chunked content and no content remaining
|
|
if (this->contentChunked && this->contentRemaining == 0)
|
|
{
|
|
// Read length of next chunk
|
|
MEM_CONTEXT_TEMP_BEGIN()
|
|
{
|
|
this->contentRemaining = cvtZToUInt64Base(strPtr(strTrim(ioReadLine(tlsClientIoRead(this->tls)))), 16);
|
|
}
|
|
MEM_CONTEXT_TEMP_END();
|
|
|
|
// If content remaining is still zero then eof
|
|
if (this->contentRemaining == 0)
|
|
this->contentEof = true;
|
|
}
|
|
|
|
// Read if there is content remaining
|
|
if (this->contentRemaining > 0)
|
|
{
|
|
// If the buffer is larger than the content that needs to read then limit the buffer size so the read won't block
|
|
// or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is defined as less than
|
|
// 2^32 so content remaining can't be more than that.
|
|
if (bufRemains(buffer) > this->contentRemaining)
|
|
bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining));
|
|
|
|
this->contentRemaining -= ioRead(tlsClientIoRead(this->tls), buffer);
|
|
|
|
// Clear limit (this works even if the limit was not set and it is easier than checking)
|
|
bufLimitClear(buffer);
|
|
}
|
|
|
|
// If no content remaining
|
|
if (this->contentRemaining == 0)
|
|
{
|
|
// If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back
|
|
// around to check.
|
|
if (this->contentChunked)
|
|
{
|
|
ioReadLine(tlsClientIoRead(this->tls));
|
|
}
|
|
// If total content size was provided then this is eof
|
|
else
|
|
this->contentEof = true;
|
|
}
|
|
}
|
|
while (!bufFull(buffer) && !this->contentEof);
|
|
|
|
// If the server notified that it would close the connection after sending content then close the client side
|
|
if (this->contentEof && this->closeOnContentEof)
|
|
tlsClientClose(this->tls);
|
|
}
|
|
|
|
FUNCTION_DEBUG_RESULT(SIZE, (size_t)actualBytes);
|
|
}
|
|
|
|
/***********************************************************************************************************************************
|
|
Has all content been read?
|
|
***********************************************************************************************************************************/
|
|
static bool
|
|
httpClientEof(const HttpClient *this)
|
|
{
|
|
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
|
FUNCTION_DEBUG_PARAM(HTTP_CLIENT, this);
|
|
|
|
FUNCTION_TEST_ASSERT(this != NULL);
|
|
FUNCTION_DEBUG_END();
|
|
|
|
FUNCTION_DEBUG_RESULT(BOOL, this->contentEof);
|
|
}
|
|
|
|
/***********************************************************************************************************************************
|
|
New object
|
|
***********************************************************************************************************************************/
|
|
HttpClient *
|
|
httpClientNew(
|
|
const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
|
|
{
|
|
FUNCTION_DEBUG_BEGIN(logLevelDebug)
|
|
FUNCTION_DEBUG_PARAM(STRING, host);
|
|
FUNCTION_DEBUG_PARAM(UINT, port);
|
|
FUNCTION_DEBUG_PARAM(TIME_MSEC, timeout);
|
|
FUNCTION_DEBUG_PARAM(BOOL, verifyPeer);
|
|
FUNCTION_DEBUG_PARAM(STRING, caFile);
|
|
FUNCTION_DEBUG_PARAM(STRING, caPath);
|
|
|
|
FUNCTION_TEST_ASSERT(host != NULL);
|
|
FUNCTION_DEBUG_END();
|
|
|
|
HttpClient *this = NULL;
|
|
|
|
MEM_CONTEXT_NEW_BEGIN("HttpClient")
|
|
{
|
|
// Allocate state and set context
|
|
this = memNew(sizeof(HttpClient));
|
|
this->memContext = MEM_CONTEXT_NEW();
|
|
|
|
this->timeout = timeout;
|
|
this->tls = tlsClientNew(host, port, timeout, verifyPeer, caFile, caPath);
|
|
}
|
|
MEM_CONTEXT_NEW_END();
|
|
|
|
FUNCTION_DEBUG_RESULT(HTTP_CLIENT, this);
|
|
}
|
|
|
|
/***********************************************************************************************************************************
|
|
Perform a request
|
|
***********************************************************************************************************************************/
|
|
Buffer *
|
|
httpClientRequest(
|
|
HttpClient *this, const String *verb, const String *uri, const HttpQuery *query, const HttpHeader *requestHeader,
|
|
bool returnContent)
|
|
{
|
|
FUNCTION_DEBUG_BEGIN(logLevelDebug)
|
|
FUNCTION_DEBUG_PARAM(HTTP_CLIENT, this);
|
|
FUNCTION_DEBUG_PARAM(STRING, verb);
|
|
FUNCTION_DEBUG_PARAM(STRING, uri);
|
|
FUNCTION_DEBUG_PARAM(HTTP_QUERY, query);
|
|
FUNCTION_DEBUG_PARAM(HTTP_HEADER, requestHeader);
|
|
|
|
FUNCTION_TEST_ASSERT(this != NULL);
|
|
FUNCTION_TEST_ASSERT(verb != NULL);
|
|
FUNCTION_TEST_ASSERT(uri != NULL);
|
|
FUNCTION_DEBUG_END();
|
|
|
|
// Buffer for returned content
|
|
Buffer *result = NULL;
|
|
|
|
MEM_CONTEXT_TEMP_BEGIN()
|
|
{
|
|
bool complete = false;
|
|
bool retry;
|
|
Wait *wait = this->timeout > 0 ? waitNew(this->timeout) : NULL;
|
|
|
|
do
|
|
{
|
|
// Assume there will be no retry
|
|
retry = false;
|
|
|
|
// Free the read interface
|
|
ioReadFree(this->ioRead);
|
|
this->ioRead = NULL;
|
|
|
|
// Free response status left over from the last request
|
|
httpHeaderFree(this->responseHeader);
|
|
this->responseHeader = NULL;
|
|
strFree(this->responseMessage);
|
|
this->responseMessage = NULL;
|
|
|
|
// Reset all content info
|
|
this->contentChunked = false;
|
|
this->contentSize = 0;
|
|
this->contentRemaining = 0;
|
|
this->closeOnContentEof = false;
|
|
this->contentEof = true;
|
|
|
|
TRY_BEGIN()
|
|
{
|
|
tlsClientOpen(this->tls);
|
|
|
|
// Write the request
|
|
String *queryStr = httpQueryRender(query);
|
|
|
|
ioWriteLine(
|
|
tlsClientIoWrite(this->tls),
|
|
strNewFmt(
|
|
"%s %s%s%s " HTTP_VERSION "\r", strPtr(verb), strPtr(httpUriEncode(uri, true)), queryStr == NULL ? "" : "?",
|
|
queryStr == NULL ? "" : strPtr(queryStr)));
|
|
|
|
// Write headers
|
|
if (requestHeader != NULL)
|
|
{
|
|
const StringList *headerList = httpHeaderList(requestHeader);
|
|
|
|
for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
|
|
{
|
|
const String *headerKey = strLstGet(headerList, headerIdx);
|
|
ioWriteLine(
|
|
tlsClientIoWrite(this->tls),
|
|
strNewFmt("%s:%s\r", strPtr(headerKey), strPtr(httpHeaderGet(requestHeader, headerKey))));
|
|
}
|
|
}
|
|
|
|
// Write out blank line and close the write so it flushes
|
|
ioWriteLine(tlsClientIoWrite(this->tls), CR_STR);
|
|
ioWriteFlush(tlsClientIoWrite(this->tls));
|
|
|
|
// Read status and make sure it starts with the correct http version
|
|
String *status = strTrim(ioReadLine(tlsClientIoRead(this->tls)));
|
|
|
|
if (!strBeginsWith(status, HTTP_VERSION_STR))
|
|
THROW_FMT(FormatError, "http version of response '%s' must be " HTTP_VERSION, strPtr(status));
|
|
|
|
// Now read the response code and message
|
|
status = strSub(status, sizeof(HTTP_VERSION));
|
|
|
|
int spacePos = strChr(status, ' ');
|
|
|
|
if (spacePos < 0)
|
|
THROW_FMT(FormatError, "response status '%s' must have a space", strPtr(status));
|
|
|
|
this->responseCode = cvtZToUInt(strPtr(strTrim(strSubN(status, 0, (size_t)spacePos))));
|
|
|
|
MEM_CONTEXT_BEGIN(this->memContext)
|
|
{
|
|
this->responseMessage = strSub(status, (size_t)spacePos + 1);
|
|
}
|
|
MEM_CONTEXT_END();
|
|
|
|
// Read headers
|
|
MEM_CONTEXT_BEGIN(this->memContext)
|
|
{
|
|
this->responseHeader = httpHeaderNew(NULL);
|
|
}
|
|
MEM_CONTEXT_END();
|
|
|
|
do
|
|
{
|
|
// Read the next header
|
|
String *header = strTrim(ioReadLine(tlsClientIoRead(this->tls)));
|
|
|
|
// If the header is empty then we have reached the end of the headers
|
|
if (strSize(header) == 0)
|
|
break;
|
|
|
|
// Split the header and store it
|
|
int colonPos = strChr(header, ':');
|
|
|
|
if (colonPos < 0)
|
|
THROW_FMT(FormatError, "header '%s' missing colon", strPtr(strTrim(header)));
|
|
|
|
String *headerKey = strLower(strTrim(strSubN(header, 0, (size_t)colonPos)));
|
|
String *headerValue = strTrim(strSub(header, (size_t)colonPos + 1));
|
|
|
|
httpHeaderAdd(this->responseHeader, headerKey, headerValue);
|
|
|
|
// Read transfer encoding (only chunked is supported)
|
|
if (strEq(headerKey, HTTP_HEADER_TRANSFER_ENCODING_STR))
|
|
{
|
|
// Error if transfer encoding is not chunked
|
|
if (!strEq(headerValue, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR))
|
|
{
|
|
THROW_FMT(
|
|
FormatError, "only '%s' is supported for '%s' header", HTTP_VALUE_TRANSFER_ENCODING_CHUNKED,
|
|
HTTP_HEADER_TRANSFER_ENCODING);
|
|
}
|
|
|
|
this->contentChunked = true;
|
|
}
|
|
|
|
// Read content size
|
|
if (strEq(headerKey, HTTP_HEADER_CONTENT_LENGTH_STR))
|
|
{
|
|
this->contentSize = cvtZToUInt64(strPtr(headerValue));
|
|
this->contentRemaining = this->contentSize;
|
|
}
|
|
|
|
// If the server notified of a closed connection then close the client connection after reading content. This
|
|
// prevents doing a retry on the next request when using the closed connection.
|
|
if (strEq(headerKey, HTTP_HEADER_CONNECTION_STR) && strEq(headerValue, HTTP_VALUE_CONNECTION_CLOSE_STR))
|
|
this->closeOnContentEof = true;
|
|
}
|
|
while (1);
|
|
|
|
// Error if transfer encoding and content length are both set
|
|
if (this->contentChunked && this->contentSize > 0)
|
|
{
|
|
THROW_FMT(
|
|
FormatError, "'%s' and '%s' headers are both set", HTTP_HEADER_TRANSFER_ENCODING,
|
|
HTTP_HEADER_CONTENT_LENGTH);
|
|
}
|
|
|
|
// If content chunked or content length > 0 then there is content to read
|
|
if (this->contentChunked || this->contentSize > 0)
|
|
{
|
|
this->contentEof = false;
|
|
|
|
// If all content should be returned from this function then read the buffer. Also read the reponse if there
|
|
// has been an error.
|
|
if (returnContent || httpClientResponseCode(this) != 200)
|
|
{
|
|
result = bufNew(0);
|
|
|
|
do
|
|
{
|
|
bufResize(result, bufSize(result) + ioBufferSize());
|
|
httpClientRead(this, result);
|
|
}
|
|
while (!httpClientEof(this));
|
|
}
|
|
// Else create the read interface
|
|
else
|
|
{
|
|
MEM_CONTEXT_BEGIN(this->memContext)
|
|
{
|
|
this->ioRead = ioReadNewP(
|
|
this, .eof = (IoReadInterfaceEof)httpClientEof, .read = (IoReadInterfaceRead)httpClientRead);
|
|
ioReadOpen(this->ioRead);
|
|
}
|
|
MEM_CONTEXT_END();
|
|
}
|
|
}
|
|
// If the server notified that it would close the connection after sending content then close the client side
|
|
else if (this->closeOnContentEof)
|
|
tlsClientClose(this->tls);
|
|
|
|
// Retry when reponse code is 5xx. These errors generally represent a server error for a request that looks valid.
|
|
// There are a few errors that might be permanently fatal but they are rare and it seems best not to try and pick
|
|
// and choose errors in this class to retry.
|
|
if (httpClientResponseCode(this) / 100 == HTTP_RESPONSE_CODE_RETRY_CLASS)
|
|
THROW_FMT(ServiceError, "[%u] %s", httpClientResponseCode(this), strPtr(httpClientResponseMessage(this)));
|
|
|
|
// Request was successful
|
|
complete = true;
|
|
}
|
|
CATCH_ANY()
|
|
{
|
|
// Retry if wait time has not expired
|
|
if (wait != NULL && waitMore(wait))
|
|
{
|
|
LOG_DEBUG("retry %s: %s", errorTypeName(errorType()), errorMessage());
|
|
retry = true;
|
|
}
|
|
|
|
tlsClientClose(this->tls);
|
|
}
|
|
TRY_END();
|
|
}
|
|
while (!complete && retry);
|
|
|
|
if (!complete)
|
|
RETHROW();
|
|
|
|
// Move the result buffer (if any) to the parent context
|
|
bufMove(result, MEM_CONTEXT_OLD());
|
|
}
|
|
MEM_CONTEXT_TEMP_END();
|
|
|
|
FUNCTION_DEBUG_RESULT(BUFFER, result);
|
|
}
|
|
|
|
/***********************************************************************************************************************************
|
|
Get read interface
|
|
***********************************************************************************************************************************/
|
|
IoRead *
|
|
httpClientIoRead(const HttpClient *this)
|
|
{
|
|
FUNCTION_TEST_BEGIN();
|
|
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
|
|
|
|
FUNCTION_TEST_ASSERT(this != NULL);
|
|
FUNCTION_TEST_END();
|
|
|
|
FUNCTION_TEST_RESULT(IO_READ, this->ioRead);
|
|
}
|
|
|
|
/***********************************************************************************************************************************
|
|
Get the response code
|
|
***********************************************************************************************************************************/
|
|
unsigned int
|
|
httpClientResponseCode(const HttpClient *this)
|
|
{
|
|
FUNCTION_TEST_BEGIN();
|
|
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
|
|
|
|
FUNCTION_TEST_ASSERT(this != NULL);
|
|
FUNCTION_TEST_END();
|
|
|
|
FUNCTION_TEST_RESULT(UINT, this->responseCode);
|
|
}
|
|
|
|
/***********************************************************************************************************************************
|
|
Get the response headers
|
|
***********************************************************************************************************************************/
|
|
const HttpHeader *
|
|
httpClientReponseHeader(const HttpClient *this)
|
|
{
|
|
FUNCTION_TEST_BEGIN();
|
|
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
|
|
|
|
FUNCTION_TEST_ASSERT(this != NULL);
|
|
FUNCTION_TEST_END();
|
|
|
|
FUNCTION_TEST_RESULT(HTTP_HEADER, this->responseHeader);
|
|
}
|
|
|
|
/***********************************************************************************************************************************
|
|
Get the response message
|
|
***********************************************************************************************************************************/
|
|
const String *
|
|
httpClientResponseMessage(const HttpClient *this)
|
|
{
|
|
FUNCTION_TEST_BEGIN();
|
|
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
|
|
|
|
FUNCTION_TEST_ASSERT(this != NULL);
|
|
FUNCTION_TEST_END();
|
|
|
|
FUNCTION_TEST_RESULT(STRING, this->responseMessage);
|
|
}
|
|
|
|
/***********************************************************************************************************************************
|
|
Free the object
|
|
***********************************************************************************************************************************/
|
|
void
|
|
httpClientFree(HttpClient *this)
|
|
{
|
|
FUNCTION_TEST_BEGIN();
|
|
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
|
|
FUNCTION_TEST_END();
|
|
|
|
if (this != NULL)
|
|
memContextFree(this->memContext);
|
|
|
|
FUNCTION_TEST_RESULT_VOID();
|
|
}
|