db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Add user-agent to HTTP requests.

Browse Source
This commit is contained in:
David Steele 2020-08-18 10:01:24 -04:00 committed by GitHub
parent 4391497a05
commit de0f8c2654
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 52 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doc/xml/release.xml
View File

@ -111,6 +111,14 @@
<p>Add support for <id>HTTP/1.0</id>.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="stephen.frost"/>
</release-item-contributor-list>
<p>Add <id>user-agent</id> to <proper>HTTP</proper> requests.</p>
</release-item>
</release-development-list>
</release-core-list>

9
src/common/io/http/request.c
View File

@ -9,6 +9,7 @@ HTTP Request
#include "common/log.h"
#include "common/type/object.h"
#include "common/wait.h"
#include "version.h"
/***********************************************************************************************************************************
HTTP constants
@ -29,6 +30,7 @@ STRING_EXTERN(HTTP_HEADER_ETAG_STR, HTTP_HEADER_
STRING_EXTERN(HTTP_HEADER_DATE_STR, HTTP_HEADER_DATE);
STRING_EXTERN(HTTP_HEADER_HOST_STR, HTTP_HEADER_HOST);
STRING_EXTERN(HTTP_HEADER_LAST_MODIFIED_STR, HTTP_HEADER_LAST_MODIFIED);
#define HTTP_HEADER_USER_AGENT "user-agent"
// 5xx errors that should always be retried
#define HTTP_RESPONSE_CODE_RETRY_CLASS 5
@ -101,11 +103,12 @@ httpRequestProcess(HttpRequest *this, bool waitForResponse, bool contentCache)
{
session = httpClientOpen(this->client);
// Format the request
// Format the request and user agent
String *requestStr =
strNewFmt(
"%s %s%s%s " HTTP_VERSION CRLF_Z, strZ(this->verb), strZ(httpUriEncode(this->uri, true)),
this->query == NULL ? "" : "?", this->query == NULL ? "" : strZ(httpQueryRenderP(this->query)));
"%s %s%s%s " HTTP_VERSION CRLF_Z HTTP_HEADER_USER_AGENT ":" PROJECT_NAME "/" PROJECT_VERSION CRLF_Z,
strZ(this->verb), strZ(httpUriEncode(this->uri, true)), this->query == NULL ? "" : "?",
this->query == NULL ? "" : strZ(httpQueryRenderP(this->query)));
// Add headers
const StringList *headerList = httpHeaderList(this->header);

57
test/src/module/common/ioHttpTest.c
View File

@ -11,6 +11,12 @@ Test HTTP
#include "common/harnessFork.h"
#include "common/harnessServer.h"
/***********************************************************************************************************************************
HTTP user agent header
***********************************************************************************************************************************/
#define TEST_USER_AGENT \
HTTP_HEADER_USER_AGENT ":" PROJECT_NAME "/" PROJECT_VERSION "\r\n"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
@ -214,7 +220,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptSleep(http, 600);
hrnServerScriptClose(http);
@ -228,7 +234,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.0 200 OK\n");
hrnServerScriptClose(http);
@ -242,7 +248,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.0 200\r\n");
hrnServerScriptClose(http);
@ -256,7 +262,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1 200 OK\r\n");
hrnServerScriptClose(http);
@ -270,7 +276,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200OK\r\n");
hrnServerScriptClose(http);
@ -284,7 +290,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\n");
hrnServerScriptClose(http);
@ -298,7 +304,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\nheader-value\r\n");
hrnServerScriptClose(http);
@ -312,7 +318,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\ntransfer-encoding:bogus\r\n");
hrnServerScriptClose(http);
@ -326,7 +332,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\ntransfer-encoding:chunked\r\ncontent-length:777\r\n\r\n");
hrnServerScriptClose(http);
@ -340,7 +346,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 503 Slow Down\r\n\r\n");
hrnServerScriptClose(http);
@ -354,13 +360,15 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\nhost:myhost.com\r\n\r\n");
hrnServerScriptExpectZ(http,
"GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\n" TEST_USER_AGENT "host:myhost.com\r\n\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 500 Internal Error\r\nConnection:close\r\n\r\n");
hrnServerScriptClose(http);
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\nhost:myhost.com\r\n\r\n");
hrnServerScriptExpectZ(
http, "GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\n" TEST_USER_AGENT "host:myhost.com\r\n\r\n");
hrnServerScriptReplyZ(
http, "HTTP/1.1 200 OK\r\nkey1:0\r\n key2 : value2\r\nConnection:ack\r\ncontent-length:0\r\n\r\n");
@ -409,7 +417,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("head request with content-length but no content");
hrnServerScriptExpectZ(http, "HEAD / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "HEAD / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.0 200 OK\r\ncontent-length:380\r\n\r\n");
hrnServerScriptClose(http);
@ -427,7 +435,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "HEAD / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "HEAD / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n");
TEST_ASSIGN(response, httpRequestResponse(httpRequestNewP(client, strNew("HEAD"), strNew("/")), true), "request");
@ -441,7 +449,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("head request with connection close but no content");
hrnServerScriptExpectZ(http, "HEAD / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "HEAD / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\nConnection:close\r\n\r\n");
hrnServerScriptClose(http);
@ -459,20 +467,20 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 503 Slow Down\r\ncontent-length:3\r\nConnection:close\r\n\r\n123");
hrnServerScriptClose(http);
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(
http, "HTTP/1.1 503 Slow Down\r\nTransfer-Encoding:chunked\r\nConnection:close\r\n\r\n0\r\n\r\n");
hrnServerScriptClose(http);
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 404 Not Found\r\n\r\n");
TEST_ASSIGN(request, httpRequestNewP(client, strNew("GET"), strNew("/")), "request");
@ -492,7 +500,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error with content");
hrnServerScriptExpectZ(http, "GET /?a=b HTTP/1.1\r\nhdr1:1\r\nhdr2:2\r\n\r\n");
hrnServerScriptExpectZ(http, "GET /?a=b HTTP/1.1\r\n" TEST_USER_AGENT "hdr1:1\r\nhdr2:2\r\n\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 403 \r\ncontent-length:7\r\n\r\nCONTENT");
StringList *headerRedact = strLstNew();
@ -531,7 +539,8 @@ testRun(void)
TEST_TITLE("request with content using content-length");
hrnServerScriptExpectZ(
http, "GET /path/file%201.txt HTTP/1.1\r\ncontent-length:30\r\n\r\n012345678901234567890123456789");
http, "GET /path/file%201.txt HTTP/1.1\r\n" TEST_USER_AGENT "content-length:30\r\n\r\n"
"012345678901234567890123456789");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\nConnection:close\r\n\r\n01234567890123456789012345678901");
hrnServerScriptClose(http);
@ -556,13 +565,13 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET /path/file%201.txt HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET /path/file%201.txt HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\ncontent-length:32\r\n\r\n0123456789012345678901234567890");
hrnServerScriptClose(http);
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET /path/file%201.txt HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET /path/file%201.txt HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\ncontent-length:32\r\n\r\n01234567890123456789012345678901");
TEST_ASSIGN(
@ -574,7 +583,7 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("request with eof before content complete");
hrnServerScriptExpectZ(http, "GET /path/file%201.txt HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET /path/file%201.txt HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(http, "HTTP/1.1 200 OK\r\ncontent-length:32\r\n\r\n0123456789012345678901234567890");
hrnServerScriptClose(http);
@ -591,7 +600,7 @@ testRun(void)
hrnServerScriptAccept(http);
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n\r\n");
hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
hrnServerScriptReplyZ(
http,
"HTTP/1.1 200 OK\r\nTransfer-Encoding:chunked\r\n\r\n"

4
test/src/module/storage/azureTest.c
View File

@ -60,8 +60,8 @@ testRequest(IoWrite *write, const char *verb, const char *uri, TestRequestParam
else
strCatZ(request, uri);
// Add HTTP version
strCatZ(request, " HTTP/1.1\r\n");
// Add HTTP version and user agent
strCatZ(request, " HTTP/1.1\r\nuser-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n");
// Add authorization string
if (driver->sharedKey != NULL)

4
test/src/module/storage/s3Test.c
View File

@ -5,6 +5,7 @@ Test S3 Storage
#include "common/io/fdRead.h"
#include "common/io/fdWrite.h"
#include "version.h"
#include "common/harnessConfig.h"
#include "common/harnessFork.h"
@ -36,9 +37,10 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *uri, Test
// Get S3 driver
StorageS3 *driver = (StorageS3 *)storageDriver(s3);
// Add authorization string
// Add verb, uri, version, user-agent, and authorization string
String *request = strNewFmt(
"%s %s HTTP/1.1\r\n"
"user-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n"
"authorization:AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,"
"SignedHeaders=content-length",
verb, uri);