diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 837f3957e..984484843 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -14,6 +14,16 @@ + + + + + + +

Read HTTP content to eof when size/encoding not specified.

+ + + @@ -7917,6 +7927,11 @@ keithf4 + + kikijolicoeur + kikijolicoeur + + Kyle Nevins kyle-nevins diff --git a/src/common/io/http/client.c b/src/common/io/http/client.c index 0c267e81f..517520f4c 100644 --- a/src/common/io/http/client.c +++ b/src/common/io/http/client.c @@ -94,58 +94,68 @@ httpClientRead(THIS_VOID, Buffer *buffer, bool block) if (!this->contentEof) { - do + // If close was requested and no content specified then the server may send content up until the eof + if (this->closeOnContentEof && !this->contentChunked && this->contentSize == 0) { - // If chunked content and no content remaining - if (this->contentChunked && this->contentRemaining == 0) - { - // Read length of next chunk - MEM_CONTEXT_TEMP_BEGIN() - { - this->contentRemaining = cvtZToUInt64Base(strPtr(strTrim(ioReadLine(tlsClientIoRead(this->tls)))), 16); - } - MEM_CONTEXT_TEMP_END(); - - // If content remaining is still zero then eof - if (this->contentRemaining == 0) - this->contentEof = true; - } - - // Read if there is content remaining - if (this->contentRemaining > 0) - { - // If the buffer is larger than the content that needs to be read then limit the buffer size so the read won't block - // or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is defined as less than - // 2^32 so content remaining can't be more than that. - if (bufRemains(buffer) > this->contentRemaining) - bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining)); - - actualBytes = bufRemains(buffer); - this->contentRemaining -= ioRead(tlsClientIoRead(this->tls), buffer); - - // Error if EOF but content read is not complete - if (ioReadEof(tlsClientIoRead(this->tls))) - THROW(FileReadError, "unexpected EOF reading HTTP content"); - - // Clear limit (this works even if the limit was not set and it is easier than checking) - bufLimitClear(buffer); - } - - // If no content remaining - if (this->contentRemaining == 0) - { - // If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back - // around to check. - if (this->contentChunked) - { - ioReadLine(tlsClientIoRead(this->tls)); - } - // If total content size was provided then this is eof - else - this->contentEof = true; - } + ioRead(tlsClientIoRead(this->tls), buffer); + this->contentEof = ioReadEof(tlsClientIoRead(this->tls)); + } + // Else read using specified encoding or size + else + { + do + { + // If chunked content and no content remaining + if (this->contentChunked && this->contentRemaining == 0) + { + // Read length of next chunk + MEM_CONTEXT_TEMP_BEGIN() + { + this->contentRemaining = cvtZToUInt64Base(strPtr(strTrim(ioReadLine(tlsClientIoRead(this->tls)))), 16); + } + MEM_CONTEXT_TEMP_END(); + + // If content remaining is still zero then eof + if (this->contentRemaining == 0) + this->contentEof = true; + } + + // Read if there is content remaining + if (this->contentRemaining > 0) + { + // If the buffer is larger than the content that needs to be read then limit the buffer size so the read won't + // block or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is defined as + // less than 2^32 so content remaining can't be more than that. + if (bufRemains(buffer) > this->contentRemaining) + bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining)); + + actualBytes = bufRemains(buffer); + this->contentRemaining -= ioRead(tlsClientIoRead(this->tls), buffer); + + // Error if EOF but content read is not complete + if (ioReadEof(tlsClientIoRead(this->tls))) + THROW(FileReadError, "unexpected EOF reading HTTP content"); + + // Clear limit (this works even if the limit was not set and it is easier than checking) + bufLimitClear(buffer); + } + + // If no content remaining + if (this->contentRemaining == 0) + { + // If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back + // around to check. + if (this->contentChunked) + { + ioReadLine(tlsClientIoRead(this->tls)); + } + // If total content size was provided then this is eof + else + this->contentEof = true; + } + } + while (!bufFull(buffer) && !this->contentEof); } - while (!bufFull(buffer) && !this->contentEof); // If the server notified that it would close the connection after sending content then close the client side if (this->contentEof && this->closeOnContentEof) @@ -389,7 +399,8 @@ httpClientRequest( } // Was content returned in the response? HEAD will report content but not actually return any. - bool contentExists = (this->contentChunked || this->contentSize > 0) && !strEq(verb, HTTP_VERB_HEAD_STR); + bool contentExists = + (this->contentChunked || this->contentSize > 0 || this->closeOnContentEof) && !strEq(verb, HTTP_VERB_HEAD_STR); this->contentEof = !contentExists; // If all content should be returned from this function then read the buffer. Also read the response if there has diff --git a/test/src/module/common/ioHttpTest.c b/test/src/module/common/ioHttpTest.c index d7f1f6c62..1cd2a4b70 100644 --- a/test/src/module/common/ioHttpTest.c +++ b/test/src/module/common/ioHttpTest.c @@ -165,15 +165,31 @@ testHttpServer(void) "Transfer-Encoding: chunked\r\n" "\r\n"); - // Error with content length 0 (with a few slow down errors) + // Head request with connection close but no content + harnessTlsServerExpect( + "HEAD / HTTP/1.1\r\n" + "\r\n"); + + harnessTlsServerReply( + "HTTP/1.1 200 OK\r\n" + "Connection:close\r\n" + "\r\n"); + + harnessTlsServerClose(); + + harnessTlsServerAccept(); + + // Error with content (with a few slow down errors) harnessTlsServerExpect( "GET / HTTP/1.1\r\n" "\r\n"); harnessTlsServerReply( "HTTP/1.1 503 Slow Down\r\n" + "content-length:3\r\n" "Connection:close\r\n" - "\r\n"); + "\r\n" + "123"); harnessTlsServerClose(); @@ -185,7 +201,10 @@ testHttpServer(void) harnessTlsServerReply( "HTTP/1.1 503 Slow Down\r\n" + "Transfer-Encoding:chunked\r\n" "Connection:close\r\n" + "\r\n" + "0\r\n" "\r\n"); harnessTlsServerClose(); @@ -225,7 +244,6 @@ testHttpServer(void) harnessTlsServerReply( "HTTP/1.1 200 OK\r\n" - "content-length:32\r\n" "Connection:close\r\n" "\r\n" "01234567890123456789012345678901"); @@ -476,7 +494,7 @@ testRun(void) TEST_ERROR(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), ServiceError, "[503] Slow Down"); // Request with no content - client->timeout = 500; + client->timeout = 2000; HttpHeader *headerRequest = httpHeaderNew(NULL); httpHeaderAdd(headerRequest, strNew("host"), strNew("myhost.com")); @@ -516,6 +534,17 @@ testRun(void) TEST_RESULT_STR_Z( httpHeaderToLog(httpClientResponseHeader(client)), "{transfer-encoding: 'chunked'}", " check response headers"); + // Head request with connection close but no content + TEST_RESULT_VOID( + httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), + "head request with connection close"); + TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code"); + TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", " check response message"); + TEST_RESULT_BOOL(httpClientEof(client), true, " io is eof"); + TEST_RESULT_BOOL(httpClientBusy(client), false, " client is not busy"); + TEST_RESULT_STR_Z( + httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", " check response headers"); + // Error with content length 0 TEST_RESULT_VOID( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "error with content length 0"); @@ -546,7 +575,7 @@ testRun(void) BUFSTRDEF("012345678901234567890123456789"), true), "request with content length"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close', content-length: '32'}", + httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", " check response headers"); TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901", " check response"); TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, " call internal read to check eof");