db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-05-31 22:49:46 +02:00
Code Issues Releases Activity

Add HttpClient object.

Browse Source 
A robust HTTP client with pipelining support and automatic retries.

Using a single object to make multiple requests is more efficient because requests are pipelined whenever possible. Requests are automatically retried when the connection has been closed by the server. Any 5xx response is also retried.

Only the HTTPS protocol is currently supported.
This commit is contained in:
David Steele 2018-11-21 19:11:45 -05:00
parent 1dd06a6e46
commit 72252ed2a1
12 changed files with 1648 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -15,6 +15,10 @@
<release date="XXXX-XX-XX" version="2.08dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-development-list>
<release-item>
<p>Add <code>HttpClient</code> object.</p>
</release-item>
<release-item>
<p>Add <code>TlsClient</code> object.</p>
</release-item>

16
src/Makefile
View File

@ -77,6 +77,10 @@ SRCS = \
common/io/filter/group.c \
common/io/filter/size.c \
common/io/handle.c \
common/io/http/client.c \
common/io/http/common.c \
common/io/http/header.c \
common/io/http/query.c \
common/io/io.c \
common/io/read.c \
common/io/tls/client.c \
@ -219,6 +223,18 @@ common/io/filter/size.o: common/io/filter/size.c common/debug.h common/error.aut
common/io/handle.o: common/io/handle.c common/debug.h common/error.auto.h common/error.h common/io/handle.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h
$(CC) $(CFLAGS) -c common/io/handle.c -o common/io/handle.o
common/io/http/client.o: common/io/http/client.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/common.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/tls/client.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h
$(CC) $(CFLAGS) -c common/io/http/client.c -o common/io/http/client.o
common/io/http/common.o: common/io/http/common.c common/debug.h common/error.auto.h common/error.h common/io/http/common.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h
$(CC) $(CFLAGS) -c common/io/http/common.c -o common/io/http/common.o
common/io/http/header.o: common/io/http/header.c common/debug.h common/error.auto.h common/error.h common/io/http/header.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/http/header.c -o common/io/http/header.o
common/io/http/query.o: common/io/http/query.c common/debug.h common/error.auto.h common/error.h common/io/http/common.h common/io/http/query.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/http/query.c -o common/io/http/query.o
common/io/io.o: common/io/io.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/io.h common/log.h common/logLevel.h common/stackTrace.h common/type/convert.h
$(CC) $(CFLAGS) -c common/io/io.c -o common/io/io.o

493
src/common/io/http/client.c Normal file
View File

@ -0,0 +1,493 @@
/***********************************************************************************************************************************
Http Client
***********************************************************************************************************************************/
#include "common/assert.h"
#include "common/debug.h"
#include "common/io/http/client.h"
#include "common/io/http/common.h"
#include "common/io/io.h"
#include "common/io/read.intern.h"
#include "common/io/tls/client.h"
#include "common/log.h"
#include "common/wait.h"
/***********************************************************************************************************************************
Http constants
***********************************************************************************************************************************/
#define HTTP_VERSION "HTTP/1.1"
STRING_STATIC(HTTP_VERSION_STR, HTTP_VERSION);
STRING_EXTERN(HTTP_VERB_GET_STR, HTTP_VERB_GET);
#define HTTP_HEADER_CONNECTION "connection"
STRING_STATIC(HTTP_HEADER_CONNECTION_STR, HTTP_HEADER_CONNECTION);
STRING_EXTERN(HTTP_HEADER_CONTENT_LENGTH_STR, HTTP_HEADER_CONTENT_LENGTH);
#define HTTP_HEADER_TRANSFER_ENCODING "transfer-encoding"
STRING_STATIC(HTTP_HEADER_TRANSFER_ENCODING_STR, HTTP_HEADER_TRANSFER_ENCODING);
#define HTTP_VALUE_CONNECTION_CLOSE "close"
STRING_STATIC(HTTP_VALUE_CONNECTION_CLOSE_STR, HTTP_VALUE_CONNECTION_CLOSE);
#define HTTP_VALUE_TRANSFER_ENCODING_CHUNKED "chunked"
STRING_STATIC(HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED);
// 5xx errors that should always be retried
#define HTTP_RESPONSE_CODE_RETRY_CLASS 5
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct HttpClient
{
MemContext *memContext; // Mem context
TimeMSec timeout; // Request timeout
TlsClient *tls; // Tls client
IoRead *ioRead; // Read io interface
unsigned int responseCode; // Response code (e.g. 200, 404)
String *responseMessage; // Response message e.g. (OK, Not Found)
HttpHeader *responseHeader; // Response headers
bool contentChunked; // Is the reponse content chunked?
uint64_t contentSize; // Content size (ignored for chunked)
uint64_t contentRemaining; // Content remaining (per chunk if chunked)
bool closeOnContentEof; // Will server close after content is sent?
bool contentEof; // Has all content been read?
};
/***********************************************************************************************************************************
Read content
***********************************************************************************************************************************/
static size_t
httpClientRead(HttpClient *this, Buffer *buffer)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(HTTP_CLIENT, this);
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(buffer != NULL);
FUNCTION_TEST_ASSERT(!bufFull(buffer));
FUNCTION_DEBUG_END();
// Read if EOF has not been reached
size_t actualBytes = 0;
if (!this->contentEof)
{
do
{
// If chunked content and no content remaining
if (this->contentChunked && this->contentRemaining == 0)
{
// Read length of next chunk
MEM_CONTEXT_TEMP_BEGIN()
{
this->contentRemaining = cvtZToUInt64Base(strPtr(strTrim(ioReadLine(tlsClientIoRead(this->tls)))), 16);
}
MEM_CONTEXT_TEMP_END();
// If content remaining is still zero then eof
if (this->contentRemaining == 0)
this->contentEof = true;
}
// Read if there is content remaining
if (this->contentRemaining > 0)
{
// If the buffer is larger than the content that needs to read then limit the buffer size so the read won't block
// or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is defined as less than
// 2^32 so content remaining can't be more than that.
if (bufRemains(buffer) > this->contentRemaining)
bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining));
this->contentRemaining -= ioRead(tlsClientIoRead(this->tls), buffer);
// Clear limit (this works even if the limit was not set and it is easier than checking)
bufLimitClear(buffer);
}
// If no content remaining
if (this->contentRemaining == 0)
{
// If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back
// around to check.
if (this->contentChunked)
{
ioReadLine(tlsClientIoRead(this->tls));
}
// If total content size was provided then this is eof
else
this->contentEof = true;
}
}
while (!bufFull(buffer) && !this->contentEof);
// If the server notified that it would close the connection after sending content then close the client side
if (this->contentEof && this->closeOnContentEof)
tlsClientClose(this->tls);
}
FUNCTION_DEBUG_RESULT(SIZE, (size_t)actualBytes);
}
/***********************************************************************************************************************************
Has all content been read?
***********************************************************************************************************************************/
static bool
httpClientEof(const HttpClient *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(HTTP_CLIENT, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_DEBUG_END();
FUNCTION_DEBUG_RESULT(BOOL, this->contentEof);
}
/***********************************************************************************************************************************
New object
***********************************************************************************************************************************/
HttpClient *
httpClientNew(
const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
{
FUNCTION_DEBUG_BEGIN(logLevelDebug)
FUNCTION_DEBUG_PARAM(STRING, host);
FUNCTION_DEBUG_PARAM(UINT, port);
FUNCTION_DEBUG_PARAM(TIME_MSEC, timeout);
FUNCTION_DEBUG_PARAM(BOOL, verifyPeer);
FUNCTION_DEBUG_PARAM(STRING, caFile);
FUNCTION_DEBUG_PARAM(STRING, caPath);
FUNCTION_TEST_ASSERT(host != NULL);
FUNCTION_DEBUG_END();
HttpClient *this = NULL;
MEM_CONTEXT_NEW_BEGIN("HttpClient")
{
// Allocate state and set context
this = memNew(sizeof(HttpClient));
this->memContext = MEM_CONTEXT_NEW();
this->timeout = timeout;
this->tls = tlsClientNew(host, port, timeout, verifyPeer, caFile, caPath);
}
MEM_CONTEXT_NEW_END();
FUNCTION_DEBUG_RESULT(HTTP_CLIENT, this);
}
/***********************************************************************************************************************************
Perform a request
***********************************************************************************************************************************/
Buffer *
httpClientRequest(
HttpClient *this, const String *verb, const String *uri, const HttpQuery *query, const HttpHeader *requestHeader,
bool returnContent)
{
FUNCTION_DEBUG_BEGIN(logLevelDebug)
FUNCTION_DEBUG_PARAM(HTTP_CLIENT, this);
FUNCTION_DEBUG_PARAM(STRING, verb);
FUNCTION_DEBUG_PARAM(STRING, uri);
FUNCTION_DEBUG_PARAM(HTTP_QUERY, query);
FUNCTION_DEBUG_PARAM(HTTP_HEADER, requestHeader);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(verb != NULL);
FUNCTION_TEST_ASSERT(uri != NULL);
FUNCTION_DEBUG_END();
// Buffer for returned content
Buffer *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
bool complete = false;
bool retry;
Wait *wait = this->timeout > 0 ? waitNew(this->timeout) : NULL;
do
{
// Assume there will be no retry
retry = false;
// Free the read interface
ioReadFree(this->ioRead);
this->ioRead = NULL;
// Free response status left over from the last request
httpHeaderFree(this->responseHeader);
this->responseHeader = NULL;
strFree(this->responseMessage);
this->responseMessage = NULL;
// Reset all content info
this->contentChunked = false;
this->contentSize = 0;
this->contentRemaining = 0;
this->closeOnContentEof = false;
this->contentEof = true;
TRY_BEGIN()
{
tlsClientOpen(this->tls);
// Write the request
String *queryStr = httpQueryRender(query);
ioWriteLine(
tlsClientIoWrite(this->tls),
strNewFmt(
"%s %s%s%s " HTTP_VERSION "\r", strPtr(verb), strPtr(httpUriEncode(uri, true)), queryStr == NULL ? "" : "?",
queryStr == NULL ? "" : strPtr(queryStr)));
// Write headers
if (requestHeader != NULL)
{
const StringList *headerList = httpHeaderList(requestHeader);
for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
{
const String *headerKey = strLstGet(headerList, headerIdx);
ioWriteLine(
tlsClientIoWrite(this->tls),
strNewFmt("%s:%s\r", strPtr(headerKey), strPtr(httpHeaderGet(requestHeader, headerKey))));
}
}
// Write out blank line and close the write so it flushes
ioWriteLine(tlsClientIoWrite(this->tls), CR_STR);
ioWriteFlush(tlsClientIoWrite(this->tls));
// Read status and make sure it starts with the correct http version
String *status = strTrim(ioReadLine(tlsClientIoRead(this->tls)));
if (!strBeginsWith(status, HTTP_VERSION_STR))
THROW_FMT(FormatError, "http version of response '%s' must be " HTTP_VERSION, strPtr(status));
// Now read the response code and message
status = strSub(status, sizeof(HTTP_VERSION));
int spacePos = strChr(status, ' ');
if (spacePos < 0)
THROW_FMT(FormatError, "response status '%s' must have a space", strPtr(status));
this->responseCode = cvtZToUInt(strPtr(strTrim(strSubN(status, 0, (size_t)spacePos))));
MEM_CONTEXT_BEGIN(this->memContext)
{
this->responseMessage = strSub(status, (size_t)spacePos + 1);
}
MEM_CONTEXT_END();
// Read headers
MEM_CONTEXT_BEGIN(this->memContext)
{
this->responseHeader = httpHeaderNew(NULL);
}
MEM_CONTEXT_END();
do
{
// Read the next header
String *header = strTrim(ioReadLine(tlsClientIoRead(this->tls)));
// If the header is empty then we have reached the end of the headers
if (strSize(header) == 0)
break;
// Split the header and store it
int colonPos = strChr(header, ':');
if (colonPos < 0)
THROW_FMT(FormatError, "header '%s' missing colon", strPtr(strTrim(header)));
String *headerKey = strLower(strTrim(strSubN(header, 0, (size_t)colonPos)));
String *headerValue = strTrim(strSub(header, (size_t)colonPos + 1));
httpHeaderAdd(this->responseHeader, headerKey, headerValue);
// Read transfer encoding (only chunked is supported)
if (strEq(headerKey, HTTP_HEADER_TRANSFER_ENCODING_STR))
{
// Error if transfer encoding is not chunked
if (!strEq(headerValue, HTTP_VALUE_TRANSFER_ENCODING_CHUNKED_STR))
{
THROW_FMT(
FormatError, "only '%s' is supported for '%s' header", HTTP_VALUE_TRANSFER_ENCODING_CHUNKED,
HTTP_HEADER_TRANSFER_ENCODING);
}
this->contentChunked = true;
}
// Read content size
if (strEq(headerKey, HTTP_HEADER_CONTENT_LENGTH_STR))
{
this->contentSize = cvtZToUInt64(strPtr(headerValue));
this->contentRemaining = this->contentSize;
}
// If the server notified of a closed connection then close the client connection after reading content. This
// prevents doing a retry on the next request when using the closed connection.
if (strEq(headerKey, HTTP_HEADER_CONNECTION_STR) && strEq(headerValue, HTTP_VALUE_CONNECTION_CLOSE_STR))
this->closeOnContentEof = true;
}
while (1);
// Error if transfer encoding and content length are both set
if (this->contentChunked && this->contentSize > 0)
{
THROW_FMT(
FormatError, "'%s' and '%s' headers are both set", HTTP_HEADER_TRANSFER_ENCODING,
HTTP_HEADER_CONTENT_LENGTH);
}
// If content chunked or content length > 0 then there is content to read
if (this->contentChunked || this->contentSize > 0)
{
this->contentEof = false;
// If all content should be returned from this function then read the buffer. Also read the reponse if there
// has been an error.
if (returnContent || httpClientResponseCode(this) != 200)
{
result = bufNew(0);
do
{
bufResize(result, bufSize(result) + ioBufferSize());
httpClientRead(this, result);
}
while (!httpClientEof(this));
}
// Else create the read interface
else
{
MEM_CONTEXT_BEGIN(this->memContext)
{
this->ioRead = ioReadNewP(
this, .eof = (IoReadInterfaceEof)httpClientEof, .read = (IoReadInterfaceRead)httpClientRead);
ioReadOpen(this->ioRead);
}
MEM_CONTEXT_END();
}
}
// If the server notified that it would close the connection after sending content then close the client side
else if (this->closeOnContentEof)
tlsClientClose(this->tls);
// Retry when reponse code is 5xx. These errors generally represent a server error for a request that looks valid.
// There are a few errors that might be permanently fatal but they are rare and it seems best not to try and pick
// and choose errors in this class to retry.
if (httpClientResponseCode(this) / 100 == HTTP_RESPONSE_CODE_RETRY_CLASS)
THROW_FMT(ServiceError, "[%u] %s", httpClientResponseCode(this), strPtr(httpClientResponseMessage(this)));
// Request was successful
complete = true;
}
CATCH_ANY()
{
// Retry if wait time has not expired
if (wait != NULL && waitMore(wait))
{
LOG_DEBUG("retry %s: %s", errorTypeName(errorType()), errorMessage());
retry = true;
}
tlsClientClose(this->tls);
}
TRY_END();
}
while (!complete && retry);
if (!complete)
RETHROW();
// Move the result buffer (if any) to the parent context
bufMove(result, MEM_CONTEXT_OLD());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_DEBUG_RESULT(BUFFER, result);
}
/***********************************************************************************************************************************
Get read interface
***********************************************************************************************************************************/
IoRead *
httpClientIoRead(const HttpClient *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(IO_READ, this->ioRead);
}
/***********************************************************************************************************************************
Get the response code
***********************************************************************************************************************************/
unsigned int
httpClientResponseCode(const HttpClient *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(UINT, this->responseCode);
}
/***********************************************************************************************************************************
Get the response headers
***********************************************************************************************************************************/
const HttpHeader *
httpClientReponseHeader(const HttpClient *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(HTTP_HEADER, this->responseHeader);
}
/***********************************************************************************************************************************
Get the response message
***********************************************************************************************************************************/
const String *
httpClientResponseMessage(const HttpClient *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(STRING, this->responseMessage);
}
/***********************************************************************************************************************************
Free the object
***********************************************************************************************************************************/
void
httpClientFree(HttpClient *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_CLIENT, this);
FUNCTION_TEST_END();
if (this != NULL)
memContextFree(this->memContext);
FUNCTION_TEST_RESULT_VOID();
}

72
src/common/io/http/client.h Normal file
View File

@ -0,0 +1,72 @@
/***********************************************************************************************************************************
Http Client
A robust HTTP client with pipelining support and automatic retries.
Using a single object to make multiple requests is more efficient because requests are piplelined whenever possible. Requests are
automatically retried when the connection has been closed by the server. Any 5xx response is also retried.
Only the HTTPS protocol is currently supported.
***********************************************************************************************************************************/
#ifndef COMMON_IO_HTTP_CLIENT_H
#define COMMON_IO_HTTP_CLIENT_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct HttpClient HttpClient;
#include "common/io/http/header.h"
#include "common/io/http/query.h"
#include "common/io/read.h"
#include "common/time.h"
#include "common/type/stringList.h"
/***********************************************************************************************************************************
HTTP Constants
***********************************************************************************************************************************/
#define HTTP_VERB_GET "GET"
STRING_DECLARE(HTTP_VERB_GET_STR);
#define HTTP_HEADER_CONTENT_LENGTH "content-length"
STRING_DECLARE(HTTP_HEADER_CONTENT_LENGTH_STR);
#define HTTP_RESPONSE_CODE_OK 200
#define HTTP_RESPONSE_CODE_FORBIDDEN 403
#define HTTP_RESPONSE_CODE_NOT_FOUND 404
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
HttpClient *httpClientNew(
const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
Buffer *httpClientRequest(
HttpClient *this, const String *verb, const String *uri, const HttpQuery *query, const HttpHeader *requestHeader,
bool returnContent);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
IoRead *httpClientIoRead(const HttpClient *this);
unsigned int httpClientResponseCode(const HttpClient *this);
const HttpHeader *httpClientReponseHeader(const HttpClient *this);
const String *httpClientResponseMessage(const HttpClient *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void httpClientFree(HttpClient *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_DEBUG_HTTP_CLIENT_TYPE \
HttpClient *
#define FUNCTION_DEBUG_HTTP_CLIENT_FORMAT(value, buffer, bufferSize) \
objToLog(value, "HttpClient", buffer, bufferSize)
#endif

45
src/common/io/http/common.c Normal file
View File

@ -0,0 +1,45 @@
/***********************************************************************************************************************************
Http Common
***********************************************************************************************************************************/
#include "common/debug.h"
#include "common/io/http/common.h"
/***********************************************************************************************************************************
Encode string to conform with URI specifications
If a path is being encoded then / characters won't be encoded.
***********************************************************************************************************************************/
String *
httpUriEncode(const String *uri, bool path)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, uri);
FUNCTION_TEST_PARAM(BOOL, path);
FUNCTION_TEST_END();
String *result = NULL;
// Encode if the string is not null
if (uri != NULL)
{
result = strNew("");
// Iterate all characters in the string
for (unsigned uriIdx = 0; uriIdx < strSize(uri); uriIdx++)
{
char uriChar = strPtr(uri)[uriIdx];
// These characters are reproduced verbatim
if ((uriChar >= 'A' && uriChar <= 'Z') || (uriChar >= 'a' && uriChar <= 'z') || (uriChar >= '0' && uriChar <= '9') ||
uriChar == '_' || uriChar == '-' || uriChar == '~' || uriChar == '.' || (path && uriChar == '/'))
{
strCatChr(result, uriChar);
}
// All other characters are hex-encoded
else
strCatFmt(result, "%%%02X", (unsigned char)uriChar);
}
}
FUNCTION_TEST_RESULT(STRING, result);
}

16
src/common/io/http/common.h Normal file
View File

@ -0,0 +1,16 @@
/***********************************************************************************************************************************
Http Common
Http common functions.
***********************************************************************************************************************************/
#ifndef COMMON_IO_HTTP_COMMON_H
#define COMMON_IO_HTTP_COMMON_H
#include "common/type/string.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
String *httpUriEncode(const String *uri, bool path);
#endif

218
src/common/io/http/header.c Normal file
View File

@ -0,0 +1,218 @@
/***********************************************************************************************************************************
Http Header
***********************************************************************************************************************************/
#include "common/debug.h"
#include "common/io/http/header.h"
#include "common/memContext.h"
#include "common/type/keyValue.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct HttpHeader
{
MemContext *memContext; // Mem context
const StringList *redactList; // List of headers to redact during logging
KeyValue *kv; // KeyValue store
};
/***********************************************************************************************************************************
New object
***********************************************************************************************************************************/
HttpHeader *
httpHeaderNew(const StringList *redactList)
{
FUNCTION_TEST_VOID();
HttpHeader *this = NULL;
MEM_CONTEXT_NEW_BEGIN("HttpHeader")
{
// Allocate state and set context
this = memNew(sizeof(HttpHeader));
this->memContext = MEM_CONTEXT_NEW();
this->redactList = redactList;
this->kv = kvNew();
}
MEM_CONTEXT_NEW_END();
FUNCTION_TEST_RESULT(HTTP_HEADER, this);
}
/***********************************************************************************************************************************
Add a header
***********************************************************************************************************************************/
HttpHeader *
httpHeaderAdd(HttpHeader *this, const String *key, const String *value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_HEADER, this);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(key != NULL);
FUNCTION_TEST_ASSERT(value != NULL);
FUNCTION_TEST_END();
MEM_CONTEXT_BEGIN(this->memContext)
{
// Make sure the key does not already exist
Variant *keyVar = varNewStr(key);
if (kvGet(this->kv, keyVar) != NULL)
THROW_FMT(AssertError, "key '%s' already exists", strPtr(key));
// Store the key
kvPut(this->kv, keyVar, varNewStr(value));
}
MEM_CONTEXT_END();
FUNCTION_TEST_RESULT(HTTP_HEADER, this);
}
/***********************************************************************************************************************************
Get a value using the key
***********************************************************************************************************************************/
const String *
httpHeaderGet(const HttpHeader *this, const String *key)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_HEADER, this);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(key != NULL);
FUNCTION_TEST_END();
String *result = NULL;
MEM_CONTEXT_BEGIN(this->memContext)
{
result = varStr(kvGet(this->kv, varNewStr(key)));
}
MEM_CONTEXT_END();
FUNCTION_TEST_RESULT(STRING, result);
}
/***********************************************************************************************************************************
Get list of keys
***********************************************************************************************************************************/
StringList *
httpHeaderList(const HttpHeader *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_HEADER, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(STRING_LIST, strLstSort(strLstNewVarLst(kvKeyList(this->kv)), sortOrderAsc));
}
/***********************************************************************************************************************************
Move object to a new mem context
***********************************************************************************************************************************/
HttpHeader *
httpHeaderMove(HttpHeader *this, MemContext *parentNew)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_HEADER, this);
FUNCTION_TEST_PARAM(MEM_CONTEXT, parentNew);
FUNCTION_TEST_ASSERT(parentNew != NULL);
FUNCTION_TEST_END();
if (this != NULL)
memContextMove(this->memContext, parentNew);
FUNCTION_TEST_RESULT(HTTP_HEADER, this);
}
/***********************************************************************************************************************************
Put a header
***********************************************************************************************************************************/
HttpHeader *
httpHeaderPut(HttpHeader *this, const String *key, const String *value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_HEADER, this);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(key != NULL);
FUNCTION_TEST_ASSERT(value != NULL);
FUNCTION_TEST_END();
MEM_CONTEXT_BEGIN(this->memContext)
{
// Store the key
kvPut(this->kv, varNewStr(key), varNewStr(value));
}
MEM_CONTEXT_END();
FUNCTION_TEST_RESULT(HTTP_HEADER, this);
}
/***********************************************************************************************************************************
Should the header be redacted when logging?
***********************************************************************************************************************************/
bool
httpHeaderRedact(const HttpHeader *this, const String *key)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_HEADER, this);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(key != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->redactList != NULL && strLstExists(this->redactList, key));
}
/***********************************************************************************************************************************
Render as string for logging
***********************************************************************************************************************************/
String *
httpHeaderToLog(const HttpHeader *this)
{
String *result = strNew("{");
const StringList *keyList = httpHeaderList(this);
for (unsigned int keyIdx = 0; keyIdx < strLstSize(keyList); keyIdx++)
{
const String *key = strLstGet(keyList, keyIdx);
if (strSize(result) != 1)
strCat(result, ", ");
if (httpHeaderRedact(this, key))
strCatFmt(result, "%s: <redacted>", strPtr(key));
else
strCatFmt(result, "%s: '%s'", strPtr(key), strPtr(httpHeaderGet(this, key)));
}
strCat(result, "}");
return result;
}
/***********************************************************************************************************************************
Free the object
***********************************************************************************************************************************/
void
httpHeaderFree(HttpHeader *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_HEADER, this);
FUNCTION_TEST_END();
if (this != NULL)
memContextFree(this->memContext);
FUNCTION_TEST_RESULT_VOID();
}

50
src/common/io/http/header.h Normal file
View File

@ -0,0 +1,50 @@
/***********************************************************************************************************************************
Http Header
Object to track HTTP headers. Headers can be marked as redacted so they are not logged.
***********************************************************************************************************************************/
#ifndef COMMON_IO_HTTP_HEADER_H
#define COMMON_IO_HTTP_HEADER_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct HttpHeader HttpHeader;
#include "common/type/stringList.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
HttpHeader *httpHeaderNew(const StringList *redactList);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
HttpHeader *httpHeaderAdd(HttpHeader *this, const String *key, const String *value);
const String *httpHeaderGet(const HttpHeader *this, const String *key);
StringList *httpHeaderList(const HttpHeader *this);
HttpHeader *httpHeaderMove(HttpHeader *this, MemContext *parentNew);
HttpHeader *httpHeaderPut(HttpHeader *this, const String *header, const String *value);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool httpHeaderRedact(const HttpHeader *this, const String *key);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void httpHeaderFree(HttpHeader *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
String *httpHeaderToLog(const HttpHeader *this);
#define FUNCTION_DEBUG_HTTP_HEADER_TYPE \
HttpHeader *
#define FUNCTION_DEBUG_HTTP_HEADER_FORMAT(value, buffer, bufferSize) \
FUNCTION_DEBUG_STRING_OBJECT_FORMAT(value, httpHeaderToLog, buffer, bufferSize)
#endif

235
src/common/io/http/query.c Normal file
View File

@ -0,0 +1,235 @@
/***********************************************************************************************************************************
Http Query
***********************************************************************************************************************************/
#include "common/debug.h"
#include "common/io/http/common.h"
#include "common/io/http/query.h"
#include "common/memContext.h"
#include "common/type/keyValue.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct HttpQuery
{
MemContext *memContext; // Mem context
KeyValue *kv; // KeyValue store
};
/***********************************************************************************************************************************
New object
***********************************************************************************************************************************/
HttpQuery *
httpQueryNew(void)
{
FUNCTION_TEST_VOID();
HttpQuery *this = NULL;
MEM_CONTEXT_NEW_BEGIN("HttpQuery")
{
// Allocate state and set context
this = memNew(sizeof(HttpQuery));
this->memContext = MEM_CONTEXT_NEW();
this->kv = kvNew();
}
MEM_CONTEXT_NEW_END();
FUNCTION_TEST_RESULT(HTTP_QUERY, this);
}
/***********************************************************************************************************************************
Add a query item
***********************************************************************************************************************************/
HttpQuery *
httpQueryAdd(HttpQuery *this, const String *key, const String *value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_QUERY, this);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(key != NULL);
FUNCTION_TEST_ASSERT(value != NULL);
FUNCTION_TEST_END();
MEM_CONTEXT_BEGIN(this->memContext)
{
// Make sure the key does not already exist
Variant *keyVar = varNewStr(key);
if (kvGet(this->kv, keyVar) != NULL)
THROW_FMT(AssertError, "key '%s' already exists", strPtr(key));
// Store the key
kvPut(this->kv, keyVar, varNewStr(value));
}
MEM_CONTEXT_END();
FUNCTION_TEST_RESULT(HTTP_QUERY, this);
}
/***********************************************************************************************************************************
Get a value using the key
***********************************************************************************************************************************/
const String *
httpQueryGet(const HttpQuery *this, const String *key)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_QUERY, this);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(key != NULL);
FUNCTION_TEST_END();
String *result = NULL;
MEM_CONTEXT_BEGIN(this->memContext)
{
result = varStr(kvGet(this->kv, varNewStr(key)));
}
MEM_CONTEXT_END();
FUNCTION_TEST_RESULT(STRING, result);
}
/***********************************************************************************************************************************
Get list of keys
***********************************************************************************************************************************/
StringList *
httpQueryList(const HttpQuery *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_QUERY, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(STRING_LIST, strLstSort(strLstNewVarLst(kvKeyList(this->kv)), sortOrderAsc));
}
/***********************************************************************************************************************************
Move object to a new mem context
***********************************************************************************************************************************/
HttpQuery *
httpQueryMove(HttpQuery *this, MemContext *parentNew)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_QUERY, this);
FUNCTION_TEST_PARAM(MEM_CONTEXT, parentNew);
FUNCTION_TEST_ASSERT(parentNew != NULL);
FUNCTION_TEST_END();
if (this != NULL)
memContextMove(this->memContext, parentNew);
FUNCTION_TEST_RESULT(HTTP_QUERY, this);
}
/***********************************************************************************************************************************
Put a query item
***********************************************************************************************************************************/
HttpQuery *
httpQueryPut(HttpQuery *this, const String *key, const String *value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_QUERY, this);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(key != NULL);
FUNCTION_TEST_ASSERT(value != NULL);
FUNCTION_TEST_END();
MEM_CONTEXT_BEGIN(this->memContext)
{
// Store the key
kvPut(this->kv, varNewStr(key), varNewStr(value));
}
MEM_CONTEXT_END();
FUNCTION_TEST_RESULT(HTTP_QUERY, this);
}
/***********************************************************************************************************************************
Render the query for inclusion in an http request
***********************************************************************************************************************************/
String *
httpQueryRender(const HttpQuery *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_QUERY, this);
FUNCTION_TEST_END();
String *result = NULL;
if (this != NULL)
{
const StringList *keyList = httpQueryList(this);
if (strLstSize(keyList) > 0)
{
result = strNew("");
MEM_CONTEXT_TEMP_BEGIN()
{
for (unsigned int keyIdx = 0; keyIdx < strLstSize(keyList); keyIdx++)
{
if (strSize(result) != 0)
strCat(result, "&");
strCatFmt(
result, "%s=%s", strPtr(strLstGet(keyList, keyIdx)),
strPtr(httpUriEncode(httpQueryGet(this, strLstGet(keyList, keyIdx)), false)));
}
}
MEM_CONTEXT_TEMP_END();
}
}
FUNCTION_TEST_RESULT(STRING, result);
}
/***********************************************************************************************************************************
Render as string for logging
***********************************************************************************************************************************/
String *
httpQueryToLog(const HttpQuery *this)
{
String *result = strNew("{");
const StringList *keyList = httpQueryList(this);
for (unsigned int keyIdx = 0; keyIdx < strLstSize(keyList); keyIdx++)
{
if (strSize(result) != 1)
strCat(result, ", ");
strCatFmt(
result, "%s: '%s'", strPtr(strLstGet(keyList, keyIdx)),
strPtr(httpQueryGet(this, strLstGet(keyList, keyIdx))));
}
strCat(result, "}");
return result;
}
/***********************************************************************************************************************************
Free the object
***********************************************************************************************************************************/
void
httpQueryFree(HttpQuery *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_QUERY, this);
FUNCTION_TEST_END();
if (this != NULL)
memContextFree(this->memContext);
FUNCTION_TEST_RESULT_VOID();
}

46
src/common/io/http/query.h Normal file
View File

@ -0,0 +1,46 @@
/***********************************************************************************************************************************
Http Query
Object to track HTTP queries and output them with proper escaping.
***********************************************************************************************************************************/
#ifndef COMMON_IO_HTTP_QUERY_H
#define COMMON_IO_HTTP_QUERY_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct HttpQuery HttpQuery;
#include "common/type/stringList.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
HttpQuery *httpQueryNew(void);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
HttpQuery *httpQueryAdd(HttpQuery *this, const String *key, const String *value);
const String *httpQueryGet(const HttpQuery *this, const String *key);
StringList *httpQueryList(const HttpQuery *this);
HttpQuery *httpQueryMove(HttpQuery *this, MemContext *parentNew);
HttpQuery *httpQueryPut(HttpQuery *this, const String *header, const String *value);
String *httpQueryRender(const HttpQuery *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void httpQueryFree(HttpQuery *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
String *httpQueryToLog(const HttpQuery *this);
#define FUNCTION_DEBUG_HTTP_QUERY_TYPE \
HttpQuery *
#define FUNCTION_DEBUG_HTTP_QUERY_FORMAT(value, buffer, bufferSize) \
FUNCTION_DEBUG_STRING_OBJECT_FORMAT(value, httpQueryToLog, buffer, bufferSize)
#endif

10
test/define.yaml
View File

@ -232,6 +232,16 @@ unit:
coverage:
common/io/tls/client: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: io-http
total: 4
coverage:
common/io/http/client: full
common/io/http/common: full
common/io/http/header: full
common/io/http/query: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: encode
total: 1

443
test/src/module/common/ioHttpTest.c Normal file
View File

@ -0,0 +1,443 @@
/***********************************************************************************************************************************
Test Http
***********************************************************************************************************************************/
#include <unistd.h>
#include "common/harnessTls.h"
#include "common/time.h"
/***********************************************************************************************************************************
Test server
***********************************************************************************************************************************/
static void
testHttpServer(void)
{
if (fork() == 0)
{
harnessTlsServerInit(TLS_TEST_PORT, TLS_CERT_TEST_CERT, TLS_CERT_TEST_KEY);
// Test no output from server
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
// Test invalid http version
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.0 200 OK\r\n");
harnessTlsServerClose();
// Test no space in status
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200OK\r\n");
harnessTlsServerClose();
// Test unexpected end of headers
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n");
harnessTlsServerClose();
// Test missing colon in header
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"header-value\r\n");
harnessTlsServerClose();
// Test invalid transfer encoding
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"transfer-encoding:bogus\r\n");
harnessTlsServerClose();
// Test content length and transfer encoding both set
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"transfer-encoding:chunked\r\n"
"content-length:777\r\n"
"\r\n");
harnessTlsServerClose();
// Test 5xx error with no retry
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 503 Slow Down\r\n"
"\r\n");
harnessTlsServerClose();
// Request with no content (with an internal error)
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\n"
"host:myhost.com\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 500 Internal Error\r\n"
"Connection:close\r\n"
"\r\n");
harnessTlsServerClose();
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\n"
"host:myhost.com\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"key1:0\r\n"
" key2 : value2\r\n"
"Connection:ack\r\n"
"\r\n");
// Error with content length 0 (with a few slow down errors)
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 503 Slow Down\r\n"
"Connection:close\r\n"
"\r\n");
harnessTlsServerClose();
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 503 Slow Down\r\n"
"Connection:close\r\n"
"\r\n");
harnessTlsServerClose();
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 404 Not Found\r\n"
"content-length:0\r\n"
"\r\n");
// Error with content
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 403 Auth Error\r\n"
"content-length:7\r\n"
"\r\n"
"CONTENT");
// Request with content
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET /path/file%201.txt HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"content-length:32\r\n"
"Connection:close\r\n"
"\r\n"
"01234567890123456789012345678901");
harnessTlsServerClose();
// Request with chunked content
harnessTlsServerAccept();
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"Transfer-Encoding:chunked\r\n"
"\r\n"
"20\r\n"
"01234567890123456789012345678901\r\n"
"10\r\n"
"0123456789012345\r\n"
"0\r\n"
"\r\n");
harnessTlsServerClose();
exit(0);
}
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("httpUriEncode"))
{
TEST_RESULT_PTR(httpUriEncode(NULL, false), NULL, "null encodes to null");
TEST_RESULT_STR(strPtr(httpUriEncode(strNew("0-9_~/A Z.az"), false)), "0-9_~%2FA%20Z.az", "non-path encoding");
TEST_RESULT_STR(strPtr(httpUriEncode(strNew("0-9_~/A Z.az"), true)), "0-9_~/A%20Z.az", "path encoding");
}
// *****************************************************************************************************************************
if (testBegin("HttpHeader"))
{
HttpHeader *header = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(header, httpHeaderNew(NULL), "new header");
TEST_RESULT_PTR(httpHeaderMove(header, MEM_CONTEXT_OLD()), header, "move to new context");
TEST_RESULT_PTR(httpHeaderMove(NULL, MEM_CONTEXT_OLD()), NULL, "move null to new context");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_PTR(httpHeaderAdd(header, strNew("key2"), strNew("value2")), header, "add header");
TEST_ERROR(httpHeaderAdd(header, strNew("key2"), strNew("value2")), AssertError, "key 'key2' already exists");
TEST_RESULT_PTR(httpHeaderPut(header, strNew("key2"), strNew("value2a")), header, "put header");
TEST_RESULT_PTR(httpHeaderAdd(header, strNew("key1"), strNew("value1")), header, "add header");
TEST_RESULT_STR(strPtr(strLstJoin(httpHeaderList(header), ", ")), "key1, key2", "header list");
TEST_RESULT_STR(strPtr(httpHeaderGet(header, strNew("key1"))), "value1", "get value");
TEST_RESULT_STR(strPtr(httpHeaderGet(header, strNew("key2"))), "value2a", "get value");
TEST_RESULT_PTR(httpHeaderGet(header, strNew(BOGUS_STR)), NULL, "get missing value");
TEST_RESULT_STR(strPtr(httpHeaderToLog(header)), "{key1: 'value1', key2: 'value2a'}", "log output");
TEST_RESULT_VOID(httpHeaderFree(header), "free header");
TEST_RESULT_VOID(httpHeaderFree(NULL), "free null header");
// Redacted headers
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(header, httpHeaderNew(strLstAddZ(strLstNew(), "secret")), "new header with redaction");
httpHeaderAdd(header, strNew("secret"), strNew("secret-value"));
httpHeaderAdd(header, strNew("public"), strNew("public-value"));
TEST_RESULT_STR(strPtr(httpHeaderToLog(header)), "{public: 'public-value', secret: <redacted>}", "log output");
}
// *****************************************************************************************************************************
if (testBegin("HttpQuery"))
{
HttpQuery *query = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(query, httpQueryNew(), "new query");
TEST_RESULT_PTR(httpQueryMove(query, MEM_CONTEXT_OLD()), query, "move to new context");
TEST_RESULT_PTR(httpQueryMove(NULL, MEM_CONTEXT_OLD()), NULL, "move null to new context");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_PTR(httpQueryRender(NULL), NULL, "null query renders null");
TEST_RESULT_PTR(httpQueryRender(query), NULL, "empty query renders null");
TEST_RESULT_PTR(httpQueryAdd(query, strNew("key2"), strNew("value2")), query, "add query");
TEST_ERROR(httpQueryAdd(query, strNew("key2"), strNew("value2")), AssertError, "key 'key2' already exists");
TEST_RESULT_PTR(httpQueryPut(query, strNew("key2"), strNew("value2a")), query, "put query");
TEST_RESULT_STR(strPtr(httpQueryRender(query)), "key2=value2a", "render one query item");
TEST_RESULT_PTR(httpQueryAdd(query, strNew("key1"), strNew("value 1?")), query, "add query");
TEST_RESULT_STR(strPtr(strLstJoin(httpQueryList(query), ", ")), "key1, key2", "query list");
TEST_RESULT_STR(strPtr(httpQueryRender(query)), "key1=value%201%3F&key2=value2a", "render two query items");
TEST_RESULT_STR(strPtr(httpQueryGet(query, strNew("key1"))), "value 1?", "get value");
TEST_RESULT_STR(strPtr(httpQueryGet(query, strNew("key2"))), "value2a", "get value");
TEST_RESULT_PTR(httpQueryGet(query, strNew(BOGUS_STR)), NULL, "get missing value");
TEST_RESULT_STR(strPtr(httpQueryToLog(query)), "{key1: 'value 1?', key2: 'value2a'}", "log output");
TEST_RESULT_VOID(httpQueryFree(query), "free query");
TEST_RESULT_VOID(httpQueryFree(NULL), "free null query");
}
// *****************************************************************************************************************************
if (testBegin("HttpClient"))
{
HttpClient *client = NULL;
ioBufferSizeSet(35);
TEST_ASSIGN(client, httpClientNew(strNew("localhost"), TLS_TEST_PORT, 500, true, NULL, NULL), "new client");
TEST_ERROR(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FileOpenError,
"unable to connect to 'localhost:9443': [111] Connection refused");
// Start http test server
testHttpServer();
// Test no output from server
TEST_ASSIGN(client, httpClientNew(strNew(TLS_TEST_HOST), TLS_TEST_PORT, 500, true, NULL, NULL), "new client");
client->timeout = 0;
TEST_ERROR(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FileReadError,
"unable to read data from '" TLS_TEST_HOST ":9443' after 500ms");
// Test invalid http version
TEST_ERROR(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError,
"http version of response 'HTTP/1.0 200 OK' must be HTTP/1.1");
// Test no space in status
TEST_ERROR(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError,
"response status '200OK' must have a space");
// Test unexpected end of headers
TEST_ERROR(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FileReadError,
"unexpected eof while reading line");
// Test missing colon in header
TEST_ERROR(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError,
"header 'header-value' missing colon");
// Test invalid transfer encoding
TEST_ERROR(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError,
"only 'chunked' is supported for 'transfer-encoding' header");
// Test content length and transfer encoding both set
TEST_ERROR(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError,
"'transfer-encoding' and 'content-length' headers are both set");
// Test 5xx error with no retry
TEST_ERROR(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), ServiceError, "[503] Slow Down");
// Request with no content
client->timeout = 500;
HttpHeader *headerRequest = httpHeaderNew(NULL);
httpHeaderAdd(headerRequest, strNew("host"), strNew("myhost.com"));
HttpQuery *query = httpQueryNew();
httpQueryAdd(query, strNew("name"), strNew("/path/A Z.txt"));
httpQueryAdd(query, strNew("type"), strNew("test"));
TEST_RESULT_VOID(
httpClientRequest(client, strNew("GET"), strNew("/"), query, headerRequest, false), "request with no content");
TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code");
TEST_RESULT_STR(strPtr(httpClientResponseMessage(client)), "OK", " check response message");
TEST_RESULT_STR(
strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{connection: 'ack', key1: '0', key2: 'value2'}",
" check response headers");
// Error with content length 0
TEST_RESULT_VOID(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), "error with content length 0");
TEST_RESULT_UINT(httpClientResponseCode(client), 404, " check response code");
TEST_RESULT_STR(strPtr(httpClientResponseMessage(client)), "Not Found", " check response message");
TEST_RESULT_STR(
strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{content-length: '0'}", " check response headers");
// Error with content
Buffer *buffer = NULL;
TEST_ASSIGN(buffer, httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), "error with content length");
TEST_RESULT_UINT(httpClientResponseCode(client), 403, " check response code");
TEST_RESULT_STR(strPtr(httpClientResponseMessage(client)), "Auth Error", " check response message");
TEST_RESULT_STR(
strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{content-length: '7'}", " check response headers");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "CONTENT", " check response");
// Request with content using content-length
ioBufferSizeSet(30);
TEST_ASSIGN(
buffer,
httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, true), "request with content length");
TEST_RESULT_STR(
strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{connection: 'close', content-length: '32'}",
" check response headers");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "01234567890123456789012345678901", " check response");
TEST_RESULT_UINT(httpClientRead(client, bufNew(1)), 0, " call internal read to check eof");
// Request with content using chunked encoding
TEST_RESULT_VOID(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), "request with chunked encoding");
TEST_RESULT_STR(
strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{transfer-encoding: 'chunked'}",
" check response headers");
buffer = bufNew(35);
TEST_RESULT_VOID(ioRead(httpClientIoRead(client), buffer), " read response");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "01234567890123456789012345678901012", " check response");
TEST_RESULT_VOID(httpClientFree(client), "free client");
TEST_RESULT_VOID(httpClientFree(NULL), "free null client");
}
FUNCTION_HARNESS_RESULT_VOID();
}