db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Read HTTP content to eof when size/encoding not specified.

Browse Source 
Generally, the content-size or content-encoding headers will be used to specify how much content should be expected.

There is a special case where the server sends 'Connection:close' without the content headers and the content may be read up until eof.

This appears to be an atypical usage but it is required by the specification.
This commit is contained in:
David Steele 2020-01-30 14:51:26 -07:00
parent 856980ae99
commit 0f8ec3e478
3 changed files with 111 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
doc/xml/release.xml
View File

@ -14,6 +14,16 @@
<release-list>
<release date="XXXX-XX-XX" version="2.24dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-bug-list>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="kikijolicoeur"/>
</release-item-contributor-list>
<p>Read HTTP content to eof when size/encoding not specified.</p>
</release-item>
</release-bug-list>
<release-feature-list>
<release-item>
<release-item-contributor-list>
@ -7917,6 +7927,11 @@
<contributor-id type="github">keithf4</contributor-id>
</contributor>
<contributor id="kikijolicoeur">
<contributor-name-display>kikijolicoeur</contributor-name-display>
<contributor-id type="github">kikijolicoeur</contributor-id>
</contributor>
<contributor id="kyle.nevins">
<contributor-name-display>Kyle Nevins</contributor-name-display>
<contributor-id type="github">kyle-nevins</contributor-id>

113
src/common/io/http/client.c
View File

@ -94,58 +94,68 @@ httpClientRead(THIS_VOID, Buffer *buffer, bool block)
if (!this->contentEof)
{
do
// If close was requested and no content specified then the server may send content up until the eof
if (this->closeOnContentEof && !this->contentChunked && this->contentSize == 0)
{
// If chunked content and no content remaining
if (this->contentChunked && this->contentRemaining == 0)
{
// Read length of next chunk
MEM_CONTEXT_TEMP_BEGIN()
{
this->contentRemaining = cvtZToUInt64Base(strPtr(strTrim(ioReadLine(tlsClientIoRead(this->tls)))), 16);
}
MEM_CONTEXT_TEMP_END();
// If content remaining is still zero then eof
if (this->contentRemaining == 0)
this->contentEof = true;
}
// Read if there is content remaining
if (this->contentRemaining > 0)
{
// If the buffer is larger than the content that needs to be read then limit the buffer size so the read won't block
// or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is defined as less than
// 2^32 so content remaining can't be more than that.
if (bufRemains(buffer) > this->contentRemaining)
bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining));
actualBytes = bufRemains(buffer);
this->contentRemaining -= ioRead(tlsClientIoRead(this->tls), buffer);
// Error if EOF but content read is not complete
if (ioReadEof(tlsClientIoRead(this->tls)))
THROW(FileReadError, "unexpected EOF reading HTTP content");
// Clear limit (this works even if the limit was not set and it is easier than checking)
bufLimitClear(buffer);
}
// If no content remaining
if (this->contentRemaining == 0)
{
// If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back
// around to check.
if (this->contentChunked)
{
ioReadLine(tlsClientIoRead(this->tls));
}
// If total content size was provided then this is eof
else
this->contentEof = true;
}
ioRead(tlsClientIoRead(this->tls), buffer);
this->contentEof = ioReadEof(tlsClientIoRead(this->tls));
}
// Else read using specified encoding or size
else
{
do
{
// If chunked content and no content remaining
if (this->contentChunked && this->contentRemaining == 0)
{
// Read length of next chunk
MEM_CONTEXT_TEMP_BEGIN()
{
this->contentRemaining = cvtZToUInt64Base(strPtr(strTrim(ioReadLine(tlsClientIoRead(this->tls)))), 16);
}
MEM_CONTEXT_TEMP_END();
// If content remaining is still zero then eof
if (this->contentRemaining == 0)
this->contentEof = true;
}
// Read if there is content remaining
if (this->contentRemaining > 0)
{
// If the buffer is larger than the content that needs to be read then limit the buffer size so the read won't
// block or read too far. Casting to size_t is safe on 32-bit because we know the max buffer size is defined as
// less than 2^32 so content remaining can't be more than that.
if (bufRemains(buffer) > this->contentRemaining)
bufLimitSet(buffer, bufSize(buffer) - (bufRemains(buffer) - (size_t)this->contentRemaining));
actualBytes = bufRemains(buffer);
this->contentRemaining -= ioRead(tlsClientIoRead(this->tls), buffer);
// Error if EOF but content read is not complete
if (ioReadEof(tlsClientIoRead(this->tls)))
THROW(FileReadError, "unexpected EOF reading HTTP content");
// Clear limit (this works even if the limit was not set and it is easier than checking)
bufLimitClear(buffer);
}
// If no content remaining
if (this->contentRemaining == 0)
{
// If chunked then consume the blank line that follows every chunk. There might be more chunk data so loop back
// around to check.
if (this->contentChunked)
{
ioReadLine(tlsClientIoRead(this->tls));
}
// If total content size was provided then this is eof
else
this->contentEof = true;
}
}
while (!bufFull(buffer) && !this->contentEof);
}
while (!bufFull(buffer) && !this->contentEof);
// If the server notified that it would close the connection after sending content then close the client side
if (this->contentEof && this->closeOnContentEof)
@ -389,7 +399,8 @@ httpClientRequest(
}
// Was content returned in the response? HEAD will report content but not actually return any.
bool contentExists = (this->contentChunked || this->contentSize > 0) && !strEq(verb, HTTP_VERB_HEAD_STR);
bool contentExists =
(this->contentChunked || this->contentSize > 0 || this->closeOnContentEof) && !strEq(verb, HTTP_VERB_HEAD_STR);
this->contentEof = !contentExists;
// If all content should be returned from this function then read the buffer. Also read the response if there has

39
test/src/module/common/ioHttpTest.c
View File

@ -165,15 +165,31 @@ testHttpServer(void)
"Transfer-Encoding: chunked\r\n"
"\r\n");
// Error with content length 0 (with a few slow down errors)
// Head request with connection close but no content
harnessTlsServerExpect(
"HEAD / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"Connection:close\r\n"
"\r\n");
harnessTlsServerClose();
harnessTlsServerAccept();
// Error with content (with a few slow down errors)
harnessTlsServerExpect(
"GET / HTTP/1.1\r\n"
"\r\n");
harnessTlsServerReply(
"HTTP/1.1 503 Slow Down\r\n"
"content-length:3\r\n"
"Connection:close\r\n"
"\r\n");
"\r\n"
"123");
harnessTlsServerClose();
@ -185,7 +201,10 @@ testHttpServer(void)
harnessTlsServerReply(
"HTTP/1.1 503 Slow Down\r\n"
"Transfer-Encoding:chunked\r\n"
"Connection:close\r\n"
"\r\n"
"0\r\n"
"\r\n");
harnessTlsServerClose();
@ -225,7 +244,6 @@ testHttpServer(void)
harnessTlsServerReply(
"HTTP/1.1 200 OK\r\n"
"content-length:32\r\n"
"Connection:close\r\n"
"\r\n"
"01234567890123456789012345678901");
@ -476,7 +494,7 @@ testRun(void)
TEST_ERROR(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), ServiceError, "[503] Slow Down");
// Request with no content
client->timeout = 500;
client->timeout = 2000;
HttpHeader *headerRequest = httpHeaderNew(NULL);
httpHeaderAdd(headerRequest, strNew("host"), strNew("myhost.com"));
@ -516,6 +534,17 @@ testRun(void)
TEST_RESULT_STR_Z(
httpHeaderToLog(httpClientResponseHeader(client)), "{transfer-encoding: 'chunked'}", " check response headers");
// Head request with connection close but no content
TEST_RESULT_VOID(
httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true),
"head request with connection close");
TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code");
TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", " check response message");
TEST_RESULT_BOOL(httpClientEof(client), true, " io is eof");
TEST_RESULT_BOOL(httpClientBusy(client), false, " client is not busy");
TEST_RESULT_STR_Z(
httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", " check response headers");
// Error with content length 0
TEST_RESULT_VOID(
httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "error with content length 0");
@ -546,7 +575,7 @@ testRun(void)
BUFSTRDEF("012345678901234567890123456789"), true),
"request with content length");
TEST_RESULT_STR_Z(
httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close', content-length: '32'}",
httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}",
" check response headers");
TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901", " check response");
TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, " call internal read to check eof");