db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-04-17 11:46:39 +02:00
Code Issues Releases Activity

Retry S3 RequestTimeTooSkewed errors instead of immediately terminating.

Browse Source 
The cause of this error seems to be that a failed request takes so long that a subsequent retry at the http level uses outdated headers.

We're not sure if pgBackRest it to blame here (in one case a kernel downgrade fixed it, in another case an incorrect network driver was the problem) so add retries to hopefully deal with the issue if it is not too persistent.  If SSL_write() has long delays before reporting an error then this will obviously affect backup performance.

Reported by sean0101n, Tim Garton, Jesper St John, Aleš Zelený.
This commit is contained in:
David Steele 2019-08-01 14:28:30 -04:00
parent 2eb3c9f95f
commit f9e1f3a798
3 changed files with 226 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
doc/xml/release.xml
View File

@ -15,6 +15,17 @@
<release date="XXXX-XX-XX" version="2.16dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-bug-list>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="sean0101n"/>
<release-item-ideator id="tim.garton"/>
<release-item-ideator id="jesper.st.john"/>
<release-item-ideator id="ales.zeleny"/>
</release-item-contributor-list>
<p>Retry <proper>S3</proper> <id>RequestTimeTooSkewed</id> errors instead of immediately terminating.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="pavel.suderevsky"/>
@ -7132,6 +7143,11 @@
<contributor-id type="github">arogozin</contributor-id>
</contributor>
<contributor id="ales.zeleny">
<contributor-name-display>Ale&amp;scaron; Zelen&amp;yacute;</contributor-name-display>
<contributor-id type="github">aleszeleny</contributor-id>
</contributor>
<contributor id="andres.freund">
<contributor-name-display>Andres Freund</contributor-name-display>
<contributor-id type="github">anarazel</contributor-id>
@ -7514,6 +7530,11 @@
<contributor-id type="github">sfrazer</contributor-id>
</contributor>
<contributor id="sean0101n">
<contributor-name-display>sean0101n</contributor-name-display>
<contributor-id type="github">sean0101n</contributor-id>
</contributor>
<contributor id="sebastien.lardiere">
<contributor-name-display>Lardi&amp;egrave;re S&amp;eacute;bastien</contributor-name-display>
<contributor-id type="github">slardiere</contributor-id>
@ -7534,6 +7555,11 @@
<contributor-id type="github">sfrost</contributor-id>
</contributor>
<contributor id="tim.garton">
<contributor-name-display>Tim Garton</contributor-name-display>
<contributor-id type="github">ralfthewise</contributor-id>
</contributor>
<contributor id="todd.vernick">
<contributor-name-display>Todd Vernick</contributor-name-display>
<contributor-id type="github">gintoddic</contributor-id>

48
src/storage/s3/storage.c
View File

@ -45,6 +45,11 @@ STRING_STATIC(S3_QUERY_PREFIX_STR, "prefix");
STRING_STATIC(S3_QUERY_VALUE_LIST_TYPE_2_STR, "2");
/***********************************************************************************************************************************
S3 errors
***********************************************************************************************************************************/
STRING_STATIC(S3_ERROR_REQUEST_TIME_TOO_SKEWED_STR, "RequestTimeTooSkewed");
/***********************************************************************************************************************************
XML tags
***********************************************************************************************************************************/
@ -249,6 +254,12 @@ storageS3Request(
ASSERT(uri != NULL);
StorageS3RequestResult result = {0};
unsigned int retryRemaining = 2;
bool done;
do
{
done = true;
MEM_CONTEXT_TEMP_BEGIN()
{
@ -284,6 +295,35 @@ storageS3Request(
// Error if the request was not successful
if (!httpClientResponseCodeOk(httpClient) &&
(!allowMissing || httpClientResponseCode(httpClient) != HTTP_RESPONSE_CODE_NOT_FOUND))
{
// If there are retries remaining and a response parse it as XML to extract the S3 error code
if (response != NULL && retryRemaining > 0)
{
// Attempt to parse the XML and extract the S3 error code
TRY_BEGIN()
{
XmlNode *error = xmlDocumentRoot(xmlDocumentNewBuf(response));
const String *errorCode = xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_CODE_STR, true));
if (strEq(errorCode, S3_ERROR_REQUEST_TIME_TOO_SKEWED_STR))
{
LOG_DEBUG(
"retry %s: %s", strPtr(errorCode),
strPtr(xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_MESSAGE_STR, true))));
retryRemaining--;
done = false;
}
}
// On failure just drop through and report the error as usual
CATCH_ANY()
{
}
TRY_END();
}
// If not done then retry instead of reporting the error
if (done)
{
// General error message
String *error = strNewFmt(
@ -334,13 +374,19 @@ storageS3Request(
THROW(ProtocolError, strPtr(error));
}
}
else
{
// On success move the buffer to the calling context
result.httpClient = httpClient;
result.responseHeader = httpHeaderMove(httpHeaderDup(httpClientReponseHeader(httpClient), NULL), MEM_CONTEXT_OLD());
result.response = bufMove(response, MEM_CONTEXT_OLD());
}
}
MEM_CONTEXT_TEMP_END();
}
while (!done);
FUNCTION_LOG_RETURN(STORAGE_S3_REQUEST_RESULT, result);
}

81
test/src/module/storage/s3Test.c
View File

@ -113,6 +113,21 @@ testS3Server(void)
// -------------------------------------------------------------------------------------------------------------------------
// File is written all at once
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD"));
harnessTlsServerReply(testS3ServerResponse(
403, "Forbidden", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<Error>"
"<Code>RequestTimeTooSkewed</Code>"
"<Message>The difference between the request time and the current time is too large.</Message>"
"<RequestTime>20190726T221748Z</RequestTime>"
"<ServerTime>2019-07-26T22:33:27Z</ServerTime>"
"<MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds>"
"<RequestId>601AA1A7F7E37AE9</RequestId>"
"<HostId>KYMys77PoloZrGCkiQRyOIl0biqdHsk4T2EdTkhzkH1l8x00D4lvv/py5uUuHwQXG9qz6NRuldQ=</HostId>"
"</Error>"));
harnessTlsServerAccept();
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD"));
harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL));
// Zero-length file
@ -209,9 +224,42 @@ testS3Server(void)
// storageDriverList()
// -------------------------------------------------------------------------------------------------------------------------
// Throw error
// Throw errors
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerReply(testS3ServerResponse(344, "Another bad status", NULL, NULL));
harnessTlsServerReply(testS3ServerResponse( 344, "Another bad status", NULL, NULL));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerReply(testS3ServerResponse(
344, "Another bad status with xml", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<Error>"
"<Code>SomeOtherCode</Code>"
"</Error>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerReply(testS3ServerResponse(
403, "Forbidden", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<Error>"
"<Code>RequestTimeTooSkewed</Code>"
"<Message>The difference between the request time and the current time is too large.</Message>"
"</Error>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerReply(testS3ServerResponse(
403, "Forbidden", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<Error>"
"<Code>RequestTimeTooSkewed</Code>"
"<Message>The difference between the request time and the current time is too large.</Message>"
"</Error>"));
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
harnessTlsServerReply(testS3ServerResponse(
403, "Forbidden", NULL,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<Error>"
"<Code>RequestTimeTooSkewed</Code>"
"<Message>The difference between the request time and the current time is too large.</Message>"
"</Error>"));
// list a file/path in root
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL));
@ -837,6 +885,35 @@ testRun(void)
"host: " S3_TEST_HOST "\n"
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date: <redacted>");
TEST_ERROR(storageListNP(s3, strNew("/")), ProtocolError,
"S3 request failed with 344: Another bad status with xml\n"
"*** URI/Query ***:\n"
"/?delimiter=%2F&list-type=2\n"
"*** Request Headers ***:\n"
"authorization: <redacted>\n"
"content-length: 0\n"
"host: " S3_TEST_HOST "\n"
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date: <redacted>\n"
"*** Response Headers ***:\n"
"content-length: 79\n"
"*** Response Content ***:\n"
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>SomeOtherCode</Code></Error>");
TEST_ERROR(storageListNP(s3, strNew("/")), ProtocolError,
"S3 request failed with 403: Forbidden\n"
"*** URI/Query ***:\n"
"/?delimiter=%2F&list-type=2\n"
"*** Request Headers ***:\n"
"authorization: <redacted>\n"
"content-length: 0\n"
"host: " S3_TEST_HOST "\n"
"x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date: <redacted>\n"
"*** Response Headers ***:\n"
"content-length: 179\n"
"*** Response Content ***:\n"
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>RequestTimeTooSkewed</Code>"
"<Message>The difference between the request time and the current time is too large.</Message></Error>");
TEST_RESULT_STR(strPtr(strLstJoin(storageListNP(s3, strNew("/")), ",")), "path1,test1.txt", "list a file/path in root");
TEST_RESULT_STR(