From d377e926c806faa8d61744e27eacfa7bf610c445 Mon Sep 17 00:00:00 2001 From: David Steele Date: Sat, 16 Mar 2019 13:07:43 +0400 Subject: [PATCH] httpClientRequest() accepts a body parameter. None of our C HTTP requests have needed to output a body, but they will with the migration of archive-push. Also, add constants that are useful when POSTing/PUTing data. --- doc/xml/release.xml | 4 +++ src/common/io/http/client.c | 16 +++++++++-- src/common/io/http/client.h | 10 ++++++- src/storage/driver/s3/storage.c | 2 +- test/src/module/common/ioHttpTest.c | 43 +++++++++++++++++------------ 5 files changed, 54 insertions(+), 21 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 667041a78..0fb640497 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -41,6 +41,10 @@

Add httpHeaderDup().

+ +

httpClientRequest() accepts a body parameter.

+ +

Make strLstDup() null-tolerant.

 diff --git a/src/common/io/http/client.c b/src/common/io/http/client.c index a2a434d2b..04baf618e 100644 --- a/src/common/io/http/client.c +++ b/src/common/io/http/client.c @@ -17,12 +17,16 @@ Http constants STRING_STATIC(HTTP_VERSION_STR, HTTP_VERSION); STRING_EXTERN(HTTP_VERB_GET_STR, HTTP_VERB_GET); +STRING_EXTERN(HTTP_VERB_POST_STR, HTTP_VERB_POST); +STRING_EXTERN(HTTP_VERB_PUT_STR, HTTP_VERB_PUT); #define HTTP_HEADER_CONNECTION "connection" STRING_STATIC(HTTP_HEADER_CONNECTION_STR, HTTP_HEADER_CONNECTION); STRING_EXTERN(HTTP_HEADER_CONTENT_LENGTH_STR, HTTP_HEADER_CONTENT_LENGTH); +STRING_EXTERN(HTTP_HEADER_CONTENT_MD5_STR, HTTP_HEADER_CONTENT_MD5); #define HTTP_HEADER_TRANSFER_ENCODING "transfer-encoding" STRING_STATIC(HTTP_HEADER_TRANSFER_ENCODING_STR, HTTP_HEADER_TRANSFER_ENCODING); +STRING_EXTERN(HTTP_HEADER_ETAG_STR, HTTP_HEADER_ETAG); #define HTTP_VALUE_CONNECTION_CLOSE "close" STRING_STATIC(HTTP_VALUE_CONNECTION_CLOSE_STR, HTTP_VALUE_CONNECTION_CLOSE); @@ -190,7 +194,7 @@ Perform a request Buffer * httpClientRequest( HttpClient *this, const String *verb, const String *uri, const HttpQuery *query, const HttpHeader *requestHeader, - bool returnContent) + const Buffer *body, bool returnContent) { FUNCTION_LOG_BEGIN(logLevelDebug) FUNCTION_LOG_PARAM(HTTP_CLIENT, this); @@ -198,6 +202,8 @@ httpClientRequest( FUNCTION_LOG_PARAM(STRING, uri); FUNCTION_LOG_PARAM(HTTP_QUERY, query); FUNCTION_LOG_PARAM(HTTP_HEADER, requestHeader); + FUNCTION_LOG_PARAM(BUFFER, body); + FUNCTION_LOG_PARAM(BOOL, returnContent); FUNCTION_LOG_END(); ASSERT(this != NULL); @@ -262,8 +268,14 @@ httpClientRequest( } } - // Write out blank line and close the write so it flushes + // Write out blank line to end the headers ioWriteLine(tlsClientIoWrite(this->tls), CR_STR); + + // Write out body if any + if (body != NULL) + ioWrite(tlsClientIoWrite(this->tls), body); + + // Flush all writes ioWriteFlush(tlsClientIoWrite(this->tls)); // Read status and make sure it starts with the correct http version diff --git a/src/common/io/http/client.h b/src/common/io/http/client.h index a8549deac..c772e704c 100644 --- a/src/common/io/http/client.h +++ b/src/common/io/http/client.h @@ -27,9 +27,17 @@ HTTP Constants ***********************************************************************************************************************************/ #define HTTP_VERB_GET "GET" STRING_DECLARE(HTTP_VERB_GET_STR); +#define HTTP_VERB_POST "POST" + STRING_DECLARE(HTTP_VERB_POST_STR); +#define HTTP_VERB_PUT "PUT" + STRING_DECLARE(HTTP_VERB_PUT_STR); #define HTTP_HEADER_CONTENT_LENGTH "content-length" STRING_DECLARE(HTTP_HEADER_CONTENT_LENGTH_STR); +#define HTTP_HEADER_CONTENT_MD5 "content-md5" + STRING_DECLARE(HTTP_HEADER_CONTENT_MD5_STR); +#define HTTP_HEADER_ETAG "etag" + STRING_DECLARE(HTTP_HEADER_ETAG_STR); #define HTTP_RESPONSE_CODE_OK 200 #define HTTP_RESPONSE_CODE_FORBIDDEN 403 @@ -46,7 +54,7 @@ Functions ***********************************************************************************************************************************/ Buffer *httpClientRequest( HttpClient *this, const String *verb, const String *uri, const HttpQuery *query, const HttpHeader *requestHeader, - bool returnContent); + const Buffer *body, bool returnContent); /*********************************************************************************************************************************** Getters diff --git a/src/storage/driver/s3/storage.c b/src/storage/driver/s3/storage.c index b39fd3883..90f8b0852 100644 --- a/src/storage/driver/s3/storage.c +++ b/src/storage/driver/s3/storage.c @@ -315,7 +315,7 @@ storageDriverS3Request( strNew("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")); // Process request - result = httpClientRequest(this->httpClient, verb, uri, query, requestHeader, returnContent); + result = httpClientRequest(this->httpClient, verb, uri, query, requestHeader, body, returnContent); // Error if the request was not successful if (httpClientResponseCode(this->httpClient) != HTTP_RESPONSE_CODE_OK && diff --git a/test/src/module/common/ioHttpTest.c b/test/src/module/common/ioHttpTest.c index 822ec7d84..97ae8ee49 100644 --- a/test/src/module/common/ioHttpTest.c +++ b/test/src/module/common/ioHttpTest.c @@ -196,7 +196,9 @@ testHttpServer(void) harnessTlsServerExpect( "GET /path/file%201.txt HTTP/1.1\r\n" - "\r\n"); + "content-length:30\r\n" + "\r\n" + "012345678901234567890123456789"); harnessTlsServerReply( "HTTP/1.1 200 OK\r\n" @@ -383,7 +385,7 @@ testRun(void) TEST_ASSIGN(client, httpClientNew(strNew("localhost"), TLS_TEST_PORT, 500, true, NULL, NULL), "new client"); TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), HostConnectError, + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), HostConnectError, "unable to connect to 'localhost:9443': [111] Connection refused"); // Start http test server @@ -394,41 +396,41 @@ testRun(void) client->timeout = 0; TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FileReadError, + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FileReadError, "unable to read data from '" TLS_TEST_HOST ":9443' after 500ms"); // Test invalid http version TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError, + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "http version of response 'HTTP/1.0 200 OK' must be HTTP/1.1"); // Test no space in status TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError, + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "response status '200OK' must have a space"); // Test unexpected end of headers TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FileReadError, + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FileReadError, "unexpected eof while reading line"); // Test missing colon in header TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError, + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "header 'header-value' missing colon"); // Test invalid transfer encoding TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError, + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "only 'chunked' is supported for 'transfer-encoding' header"); // Test content length and transfer encoding both set TEST_ERROR( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), FormatError, + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "'transfer-encoding' and 'content-length' headers are both set"); // Test 5xx error with no retry - TEST_ERROR(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), ServiceError, "[503] Slow Down"); + TEST_ERROR(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), ServiceError, "[503] Slow Down"); // Request with no content client->timeout = 500; @@ -441,7 +443,7 @@ testRun(void) httpQueryAdd(query, strNew("type"), strNew("test")); TEST_RESULT_VOID( - httpClientRequest(client, strNew("GET"), strNew("/"), query, headerRequest, false), "request with no content"); + httpClientRequest(client, strNew("GET"), strNew("/"), query, headerRequest, NULL, false), "request with no content"); TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code"); TEST_RESULT_STR(strPtr(httpClientResponseMessage(client)), "OK", " check response message"); TEST_RESULT_STR( @@ -449,7 +451,8 @@ testRun(void) " check response headers"); // Error with content length 0 - TEST_RESULT_VOID(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), "error with content length 0"); + TEST_RESULT_VOID( + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "error with content length 0"); TEST_RESULT_UINT(httpClientResponseCode(client), 404, " check response code"); TEST_RESULT_STR(strPtr(httpClientResponseMessage(client)), "Not Found", " check response message"); TEST_RESULT_STR( @@ -458,7 +461,8 @@ testRun(void) // Error with content Buffer *buffer = NULL; - TEST_ASSIGN(buffer, httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), "error with content length"); + TEST_ASSIGN( + buffer, httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "error with content length"); TEST_RESULT_UINT(httpClientResponseCode(client), 403, " check response code"); TEST_RESULT_STR(strPtr(httpClientResponseMessage(client)), "Auth Error", " check response message"); TEST_RESULT_STR( @@ -470,7 +474,11 @@ testRun(void) TEST_ASSIGN( buffer, - httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, true), "request with content length"); + httpClientRequest( + client, strNew("GET"), strNew("/path/file 1.txt"), NULL, + httpHeaderAdd(httpHeaderNew(NULL), strNew("content-length"), strNew("30")), + bufNewStr(strNew("012345678901234567890123456789")), true), + "request with content length"); TEST_RESULT_STR( strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{connection: 'close', content-length: '32'}", " check response headers"); @@ -479,7 +487,7 @@ testRun(void) // Request with eof before content complete with retry TEST_ASSIGN( - buffer, httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, true), + buffer, httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, NULL, true), "request with content length retry"); TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "01234567890123456789012345678901", " check response"); TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, " call internal read to check eof"); @@ -487,13 +495,14 @@ testRun(void) // Request with eof before content and error buffer = bufNew(32); TEST_RESULT_VOID( - httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, false), + httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, NULL, false), "request with content length error"); TEST_ERROR( ioRead(httpClientIoRead(client), buffer), FileReadError, "unexpected EOF reading HTTP content"); // Request with content using chunked encoding - TEST_RESULT_VOID(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, false), "request with chunked encoding"); + TEST_RESULT_VOID( + httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "request with chunked encoding"); TEST_RESULT_STR( strPtr(httpHeaderToLog(httpClientReponseHeader(client))), "{transfer-encoding: 'chunked'}", " check response headers");