1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-07 00:35:37 +02:00
Files
pgbackrest/src/common/io/http/request.c
David Steele c7a66ac1af Improve memory usage of mem contexts.
Each mem context can track child contexts, allocations, and a callback. Before this change memory was allocated for tracking all three even if they were not used for a particular context. This made mem contexts unsuitable for String and Variant objects since they are plentiful and need to be as small as possible.

This change allows mem contexts to be configured to track any combination of child contexts, allocations, and a callback. In addition, the mem context can be configured to track a single child context and/or allocation, which saves memory and is a common use case.

Another benefit is that Variants can own objects (e.g. KeyValue) that they encapsulate. All of this makes memory accounting simpler because mem contexts have names while allocations do not. No more memory is used than before since Variants and Strings still had to store the memory context they were originally allocated in so they could be easily freed.

Update the String and Variant objects to use this new functionality. The custom strFree() and varFree() functions are no longer required and can now be a wrapper around objFree().

Lastly, this will allow strMove() and varMove() to be implemented and used in cases where strDup() and varDup() are being used to move a String or Variant to a new context. Since this will be a bit noisy it is saved for a future commit.
2022-05-18 10:52:01 -04:00

313 lines
13 KiB
C

/***********************************************************************************************************************************
HTTP Request
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/io/http/common.h"
#include "common/io/http/request.h"
#include "common/log.h"
#include "common/stat.h"
#include "common/wait.h"
#include "version.h"
/***********************************************************************************************************************************
HTTP constants
***********************************************************************************************************************************/
STRING_EXTERN(HTTP_VERSION_STR, HTTP_VERSION);
STRING_EXTERN(HTTP_VERSION_10_STR, HTTP_VERSION_10);
STRING_EXTERN(HTTP_VERB_DELETE_STR, HTTP_VERB_DELETE);
STRING_EXTERN(HTTP_VERB_GET_STR, HTTP_VERB_GET);
STRING_EXTERN(HTTP_VERB_HEAD_STR, HTTP_VERB_HEAD);
STRING_EXTERN(HTTP_VERB_POST_STR, HTTP_VERB_POST);
STRING_EXTERN(HTTP_VERB_PUT_STR, HTTP_VERB_PUT);
STRING_EXTERN(HTTP_HEADER_AUTHORIZATION_STR, HTTP_HEADER_AUTHORIZATION);
STRING_EXTERN(HTTP_HEADER_CONTENT_LENGTH_STR, HTTP_HEADER_CONTENT_LENGTH);
STRING_EXTERN(HTTP_HEADER_CONTENT_MD5_STR, HTTP_HEADER_CONTENT_MD5);
STRING_EXTERN(HTTP_HEADER_CONTENT_RANGE_STR, HTTP_HEADER_CONTENT_RANGE);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_STR, HTTP_HEADER_CONTENT_TYPE);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL_STR, HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_JSON_STR, HTTP_HEADER_CONTENT_TYPE_JSON);
STRING_EXTERN(HTTP_HEADER_CONTENT_TYPE_XML_STR, HTTP_HEADER_CONTENT_TYPE_XML);
STRING_EXTERN(HTTP_HEADER_ETAG_STR, HTTP_HEADER_ETAG);
STRING_EXTERN(HTTP_HEADER_DATE_STR, HTTP_HEADER_DATE);
STRING_EXTERN(HTTP_HEADER_HOST_STR, HTTP_HEADER_HOST);
STRING_EXTERN(HTTP_HEADER_LAST_MODIFIED_STR, HTTP_HEADER_LAST_MODIFIED);
STRING_EXTERN(HTTP_HEADER_RANGE_STR, HTTP_HEADER_RANGE);
#define HTTP_HEADER_USER_AGENT "user-agent"
// 5xx errors that should always be retried
#define HTTP_RESPONSE_CODE_RETRY_CLASS 5
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct HttpRequest
{
HttpRequestPub pub; // Publicly accessible variables
HttpClient *client; // HTTP client
const Buffer *content; // HTTP content
HttpSession *session; // Session for async requests
};
/***********************************************************************************************************************************
Process the request
***********************************************************************************************************************************/
static HttpResponse *
httpRequestProcess(HttpRequest *this, bool waitForResponse, bool contentCache)
{
FUNCTION_LOG_BEGIN(logLevelDebug)
FUNCTION_LOG_PARAM(HTTP_REQUEST, this);
FUNCTION_LOG_PARAM(BOOL, waitForResponse);
FUNCTION_LOG_PARAM(BOOL, contentCache);
FUNCTION_LOG_END();
ASSERT(this != NULL);
// HTTP Response
HttpResponse *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
bool retry;
Wait *wait = waitNew(httpClientTimeout(this->client));
do
{
// Assume there will be no retry
retry = false;
TRY_BEGIN()
{
MEM_CONTEXT_TEMP_BEGIN()
{
HttpSession *session = NULL;
// If a session is saved then the request was already successfully sent
if (this->session != NULL)
{
session = httpSessionMove(this->session, memContextCurrent());
this->session = NULL;
}
// Else the request has not been sent yet or this is a retry
else
{
session = httpClientOpen(this->client);
// Format the request and user agent
String *requestStr =
strCatFmt(
strNew(),
"%s %s%s%s " HTTP_VERSION "\r\n" HTTP_HEADER_USER_AGENT ":" PROJECT_NAME "/" PROJECT_VERSION "\r\n",
strZ(httpRequestVerb(this)), strZ(httpRequestPath(this)), httpRequestQuery(this) == NULL ? "" : "?",
httpRequestQuery(this) == NULL ? "" : strZ(httpQueryRenderP(httpRequestQuery(this))));
// Add headers
const StringList *headerList = httpHeaderList(httpRequestHeader(this));
for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
{
const String *headerKey = strLstGet(headerList, headerIdx);
strCatFmt(
requestStr, "%s:%s\r\n", strZ(headerKey), strZ(httpHeaderGet(httpRequestHeader(this), headerKey)));
}
// Add blank line to end of headers and write the request as a buffer so secrets do not show up in logs
strCat(requestStr, CRLF_STR);
ioWrite(httpSessionIoWrite(session), BUFSTR(requestStr));
// Write out content if any
if (this->content != NULL)
ioWrite(httpSessionIoWrite(session), this->content);
// Flush all writes
ioWriteFlush(httpSessionIoWrite(session));
// If not waiting for the response then move the session to the object context
if (!waitForResponse)
this->session = httpSessionMove(session, objMemContext(this));
}
// Wait for response
if (waitForResponse)
{
result = httpResponseNew(session, httpRequestVerb(this), contentCache);
// Retry when response code is 5xx. These errors generally represent a server error for a request that
// looks valid. There are a few errors that might be permanently fatal but they are rare and it seems best
// not to try and pick and choose errors in this class to retry.
if (httpResponseCode(result) / 100 == HTTP_RESPONSE_CODE_RETRY_CLASS)
THROW_FMT(ServiceError, "[%u] %s", httpResponseCode(result), strZ(httpResponseReason(result)));
// Move response to outer temp context
httpResponseMove(result, memContextPrior());
}
}
MEM_CONTEXT_TEMP_END();
}
CATCH_ANY()
{
// Sleep and then retry unless the total wait time has expired
if (waitMore(wait))
{
LOG_DEBUG_FMT("retry %s: %s", errorTypeName(errorType()), errorMessage());
retry = true;
statInc(HTTP_STAT_RETRY_STR);
}
else
RETHROW();
}
TRY_END();
}
while (retry);
// Move response to calling context
httpResponseMove(result, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(HTTP_RESPONSE, result);
}
/**********************************************************************************************************************************/
HttpRequest *
httpRequestNew(HttpClient *client, const String *verb, const String *path, HttpRequestNewParam param)
{
FUNCTION_LOG_BEGIN(logLevelDebug)
FUNCTION_LOG_PARAM(HTTP_CLIENT, client);
FUNCTION_LOG_PARAM(STRING, verb);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_END();
ASSERT(verb != NULL);
ASSERT(path != NULL);
HttpRequest *this = NULL;
OBJ_NEW_BEGIN(HttpRequest, .childQty = MEM_CONTEXT_QTY_MAX)
{
this = OBJ_NEW_ALLOC();
*this = (HttpRequest)
{
.pub =
{
.verb = strDup(verb),
.path = strDup(path),
.query = httpQueryDupP(param.query),
.header = param.header == NULL ? httpHeaderNew(NULL) : httpHeaderDup(param.header, NULL),
},
.client = client,
.content = param.content == NULL ? NULL : bufDup(param.content),
};
}
OBJ_NEW_END();
// Send the request
httpRequestProcess(this, false, false);
statInc(HTTP_STAT_REQUEST_STR);
FUNCTION_LOG_RETURN(HTTP_REQUEST, this);
}
/**********************************************************************************************************************************/
HttpResponse *
httpRequestResponse(HttpRequest *this, bool contentCache)
{
FUNCTION_LOG_BEGIN(logLevelDebug)
FUNCTION_LOG_PARAM(HTTP_REQUEST, this);
FUNCTION_LOG_PARAM(BOOL, contentCache);
FUNCTION_LOG_END();
ASSERT(this != NULL);
FUNCTION_LOG_RETURN(HTTP_RESPONSE, httpRequestProcess(this, true, contentCache));
}
/**********************************************************************************************************************************/
void
httpRequestError(const HttpRequest *this, HttpResponse *response)
{
FUNCTION_LOG_BEGIN(logLevelTrace)
FUNCTION_LOG_PARAM(HTTP_REQUEST, this);
FUNCTION_LOG_PARAM(HTTP_RESPONSE, response);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(response != NULL);
// Error code
String *error = strCatFmt(strNew(), "HTTP request failed with %u", httpResponseCode(response));
// Add reason when present
if (strSize(httpResponseReason(response)) > 0)
strCatFmt(error, " (%s)", strZ(httpResponseReason(response)));
// Output path/query
strCatZ(error, ":\n*** Path/Query ***:");
strCatFmt(error, "\n%s %s", strZ(httpRequestVerb(this)), strZ(httpRequestPath(this)));
if (httpRequestQuery(this) != NULL)
strCatFmt(error, "?%s", strZ(httpQueryRenderP(httpRequestQuery(this), .redact = true)));
// Output request headers
const StringList *requestHeaderList = httpHeaderList(httpRequestHeader(this));
if (!strLstEmpty(requestHeaderList))
{
strCatZ(error, "\n*** Request Headers ***:");
for (unsigned int requestHeaderIdx = 0; requestHeaderIdx < strLstSize(requestHeaderList); requestHeaderIdx++)
{
const String *key = strLstGet(requestHeaderList, requestHeaderIdx);
strCatFmt(
error, "\n%s: %s", strZ(key),
httpHeaderRedact(httpRequestHeader(this), key) ? "<redacted>" : strZ(httpHeaderGet(httpRequestHeader(this), key)));
}
}
// Output response headers
const HttpHeader *responseHeader = httpResponseHeader(response);
const StringList *responseHeaderList = httpHeaderList(responseHeader);
if (!strLstEmpty(responseHeaderList))
{
strCatZ(error, "\n*** Response Headers ***:");
for (unsigned int responseHeaderIdx = 0; responseHeaderIdx < strLstSize(responseHeaderList); responseHeaderIdx++)
{
const String *key = strLstGet(responseHeaderList, responseHeaderIdx);
strCatFmt(error, "\n%s: %s", strZ(key), strZ(httpHeaderGet(responseHeader, key)));
}
}
// Add response content, if any
if (!bufEmpty(httpResponseContent(response)))
{
strCatZ(error, "\n*** Response Content ***:\n");
strCat(error, strNewBuf(httpResponseContent(response)));
}
THROW(ProtocolError, strZ(error));
}
/**********************************************************************************************************************************/
String *
httpRequestToLog(const HttpRequest *this)
{
return strNewFmt(
"{verb: %s, path: %s, query: %s, header: %s, contentSize: %zu}", strZ(httpRequestVerb(this)), strZ(httpRequestPath(this)),
httpRequestQuery(this) == NULL ? "null" : strZ(httpQueryToLog(httpRequestQuery(this))),
strZ(httpHeaderToLog(httpRequestHeader(this))), this->content == NULL ? 0 : bufUsed(this->content));
}