mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-01-18 04:58:51 +02:00
Retry when S3 returns an internal error (500).
This commit is contained in:
David Steele
parent
961f7033d1
commit
038d47bcc0
@ -202,6 +202,10 @@
|
||||
</release-feature-list>
|
||||
|
||||
<release-refactor-list>
|
||||
<release-item>
|
||||
<p>Retry when S3 returns an internal error (500).</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<p>Add <id>bIgnoreMissing</id> parameter to <code>Local->manifest()</code>.</p>
|
||||
</release-item>
|
||||
|
||||
@ -49,6 +49,7 @@ sub new
|
||||
$strOperation,
|
||||
$strHost,
|
||||
$strVerb,
|
||||
$iPort,
|
||||
$strUri,
|
||||
$hQuery,
|
||||
$hRequestHeader,
|
||||
@ -64,6 +65,7 @@ sub new
|
||||
__PACKAGE__ . '->new', \@_,
|
||||
{name => 'strHost', trace => true},
|
||||
{name => 'strVerb', trace => true},
|
||||
{name => 'iPort', optional => true, default => 443, trace => true},
|
||||
{name => 'strUri', optional => true, default => qw(/), trace => true},
|
||||
{name => 'hQuery', optional => true, trace => true},
|
||||
{name => 'hRequestHeader', optional => true, trace => true},
|
||||
@ -81,7 +83,7 @@ sub new
|
||||
eval
|
||||
{
|
||||
$oSocket = IO::Socket::SSL->new(
|
||||
PeerHost => $strHost, PeerPort => 'https', SSL_verify_mode => $bVerifySsl ? SSL_VERIFY_PEER : SSL_VERIFY_NONE,
|
||||
PeerHost => $strHost, PeerPort => $iPort, SSL_verify_mode => $bVerifySsl ? SSL_VERIFY_PEER : SSL_VERIFY_NONE,
|
||||
SSL_ca_path => $strCaPath, SSL_ca_file => $strCaFile);
|
||||
|
||||
return 1;
|
||||
|
||||
@ -46,6 +46,12 @@ use constant S3_RESPONSE_TYPE_NONE => 'none';
|
||||
use constant S3_RESPONSE_TYPE_XML => 'xml';
|
||||
push @EXPORT, qw(S3_RESPONSE_TYPE_XML);
|
||||
|
||||
use constant S3_RESPONSE_CODE_SUCCESS => 200;
|
||||
use constant S3_RESPONSE_CODE_ERROR_NOT_FOUND => 404;
|
||||
use constant S3_RESPONSE_CODE_ERROR_INTERNAL => 500;
|
||||
|
||||
use constant S3_RETRY_MAX => 2;
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
####################################################################################################################################
|
||||
@ -66,6 +72,7 @@ sub new
|
||||
$self->{strAccessKeyId},
|
||||
$self->{strSecretAccessKey},
|
||||
$self->{strHost},
|
||||
$self->{iPort},
|
||||
$self->{bVerifySsl},
|
||||
$self->{strCaPath},
|
||||
$self->{strCaFile},
|
||||
@ -80,6 +87,7 @@ sub new
|
||||
{name => 'strAccessKeyId', trace => true},
|
||||
{name => 'strSecretAccessKey', trace => true},
|
||||
{name => 'strHost', optional => true, trace => true},
|
||||
{name => 'iPort', optional => true, trace => true},
|
||||
{name => 'bVerifySsl', optional => true, default => true, trace => true},
|
||||
{name => 'strCaPath', optional => true, trace => true},
|
||||
{name => 'strCaFile', optional => true, trace => true},
|
||||
@ -128,6 +136,18 @@ sub request
|
||||
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
|
||||
);
|
||||
|
||||
# Server response
|
||||
my $oResponse;
|
||||
|
||||
# Allow retries on S3 internal failures
|
||||
my $bRetry = false;
|
||||
my $iRetryTotal = 0;
|
||||
|
||||
do
|
||||
{
|
||||
# Assume that a retry will not be attempted which is true in most cases
|
||||
$bRetry = false;
|
||||
|
||||
# Get datetime to be used for auth requests
|
||||
my $strDateTime = s3DateTime();
|
||||
|
||||
@ -143,15 +163,14 @@ sub request
|
||||
# Send the request
|
||||
my $oHttpClient = new pgBackRest::Common::Http::Client(
|
||||
$self->{strHost}, $strVerb,
|
||||
{strUri => $strUri, hQuery => $hQuery, hRequestHeader => $hHeader, rstrRequestBody => $rstrBody,
|
||||
bVerifySsl => $self->{bVerifySsl}, strCaPath => $self->{strCaPath}, strCaFile => $self->{strCaFile},
|
||||
lBufferMax => $self->{lBufferMax}});
|
||||
{iPort => $self->{iPort}, strUri => $strUri, hQuery => $hQuery, hRequestHeader => $hHeader,
|
||||
rstrRequestBody => $rstrBody, bVerifySsl => $self->{bVerifySsl}, strCaPath => $self->{strCaPath},
|
||||
strCaFile => $self->{strCaFile}, lBufferMax => $self->{lBufferMax}});
|
||||
|
||||
# Check response code
|
||||
my $iReponseCode = $oHttpClient->responseCode();
|
||||
my $oResponse;
|
||||
|
||||
if ($iReponseCode == 200)
|
||||
if ($iReponseCode == S3_RESPONSE_CODE_SUCCESS)
|
||||
{
|
||||
# Save the response headers locally
|
||||
$self->{hResponseHeader} = $oHttpClient->responseHeader();
|
||||
@ -178,14 +197,29 @@ sub request
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($iReponseCode == 404)
|
||||
# If file was not found
|
||||
if ($iReponseCode == S3_RESPONSE_CODE_ERROR_NOT_FOUND)
|
||||
{
|
||||
# If missing files should not be ignored then error
|
||||
if (!$bIgnoreMissing)
|
||||
{
|
||||
confess &log(ERROR, "unable to open '${strUri}': No such file or directory", ERROR_FILE_MISSING);
|
||||
}
|
||||
|
||||
$bRetry = false;
|
||||
}
|
||||
# Else a more serrious error
|
||||
else
|
||||
{
|
||||
# Retry for S3 internal errors
|
||||
if ($iReponseCode == S3_RESPONSE_CODE_ERROR_INTERNAL)
|
||||
{
|
||||
$iRetryTotal++;
|
||||
$bRetry = $iRetryTotal <= S3_RETRY_MAX;
|
||||
}
|
||||
|
||||
# If no retry then throw the error
|
||||
if (!$bRetry)
|
||||
{
|
||||
my $rstrResponseBody = $oHttpClient->responseBody();
|
||||
|
||||
@ -197,6 +231,9 @@ sub request
|
||||
ERROR_PROTOCOL);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
while ($bRetry);
|
||||
|
||||
# Return from function and log return values if any
|
||||
return logDebugReturn
|
||||
|
||||
@ -61,6 +61,10 @@ use constant LIB_COVER_EXE => '/usr/bin
|
||||
use constant CERT_FAKE_PATH => '/etc/fake-cert';
|
||||
use constant CERT_FAKE_CA => CERT_FAKE_PATH . '/ca.crt';
|
||||
push @EXPORT, qw(CERT_FAKE_CA);
|
||||
use constant CERT_FAKE_SERVER => CERT_FAKE_PATH . '/server.crt';
|
||||
push @EXPORT, qw(CERT_FAKE_SERVER);
|
||||
use constant CERT_FAKE_SERVER_KEY => CERT_FAKE_PATH . '/server.key';
|
||||
push @EXPORT, qw(CERT_FAKE_SERVER_KEY);
|
||||
|
||||
####################################################################################################################################
|
||||
# Container Debug - speeds container debugging by splitting each section into a separate intermediate container
|
||||
|
||||
@ -210,6 +210,15 @@ my $oTestDef =
|
||||
&TESTDEF_NAME => 's3-cert',
|
||||
&TESTDEF_TOTAL => 1,
|
||||
},
|
||||
{
|
||||
&TESTDEF_NAME => 's3-request',
|
||||
&TESTDEF_TOTAL => 2,
|
||||
|
||||
&TESTDEF_COVERAGE =>
|
||||
{
|
||||
'Storage/S3/Request' => TESTDEF_COVERAGE_PARTIAL,
|
||||
},
|
||||
},
|
||||
{
|
||||
&TESTDEF_NAME => 's3',
|
||||
&TESTDEF_TOTAL => 7,
|
||||
|
||||
178
test/lib/pgBackRestTest/Module/Storage/StorageS3RequestTest.pm
Normal file
178
test/lib/pgBackRestTest/Module/Storage/StorageS3RequestTest.pm
Normal file
@ -0,0 +1,178 @@
|
||||
####################################################################################################################################
|
||||
# S3 Request Tests
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Storage::StorageS3RequestTest;
|
||||
use parent 'pgBackRestTest::Common::RunTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use IO::Socket::SSL;
|
||||
use POSIX qw(strftime);
|
||||
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Http::Client;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Storage::S3::Request;
|
||||
|
||||
use pgBackRestTest::Common::ContainerTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# Port to use for testing
|
||||
####################################################################################################################################
|
||||
use constant HTTPS_TEST_PORT => 9443;
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServerResponse
|
||||
####################################################################################################################################
|
||||
sub httpsServerResponse
|
||||
{
|
||||
my $self = shift;
|
||||
my $iResponseCode = shift;
|
||||
my $strContent = shift;
|
||||
|
||||
# Write header
|
||||
$self->{oConnection}->write("HTTP/1.1 ${iResponseCode} GenericMessage\r\n");
|
||||
$self->{oConnection}->write(HTTP_HEADER_CONTENT_LENGTH . ': ' . (defined($strContent) ? length($strContent) : 0) . "\r\n");
|
||||
|
||||
# Write new line before content (even if there isn't any)
|
||||
$self->{oConnection}->write("\r\n");
|
||||
|
||||
# Write content
|
||||
if (defined($strContent))
|
||||
{
|
||||
$self->{oConnection}->write($strContent);
|
||||
}
|
||||
|
||||
# This will block until the connection is closed by the client
|
||||
$self->{oConnection}->read();
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServerAccept
|
||||
####################################################################################################################################
|
||||
sub httpsServerAccept
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Wait for a connection
|
||||
$self->{oConnection} = $self->{oSocketServer}->accept()
|
||||
or confess "failed to accept or handshake $!, $SSL_ERROR";
|
||||
&log(INFO, " * socket server connected");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# httpsServer
|
||||
####################################################################################################################################
|
||||
sub httpsServer
|
||||
{
|
||||
my $self = shift;
|
||||
my $fnServer = shift;
|
||||
|
||||
# Fork off the server
|
||||
if (fork() == 0)
|
||||
{
|
||||
# Run server function
|
||||
$fnServer->();
|
||||
|
||||
exit 0;
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Start the https testing server
|
||||
####################################################################################################################################
|
||||
sub initModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Open the domain socket
|
||||
$self->{oSocketServer} = IO::Socket::SSL->new(
|
||||
LocalAddr => '127.0.0.1', LocalPort => HTTPS_TEST_PORT, Listen => 1, SSL_cert_file => CERT_FAKE_SERVER,
|
||||
SSL_key_file => CERT_FAKE_SERVER_KEY)
|
||||
or confess "unable to open https server for testing: $!";
|
||||
&log(INFO, " * socket server open");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Stop the https testing server
|
||||
####################################################################################################################################
|
||||
sub cleanModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Shutdown server
|
||||
$self->{oSocketServer}->close();
|
||||
&log(INFO, " * socket server closed");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Initialize request object
|
||||
my $oS3Request = new pgBackRest::Storage::S3::Request(
|
||||
BOGUS, BOGUS, BOGUS, BOGUS, BOGUS, {strHost => '127.0.0.1', iPort => HTTPS_TEST_PORT, bVerifySsl => false});
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('success'))
|
||||
{
|
||||
$self->httpsServer(sub
|
||||
{
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(200);
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(200);
|
||||
});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request');
|
||||
$self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin('retry'))
|
||||
{
|
||||
$self->httpsServer(sub
|
||||
{
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(200);
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
|
||||
$self->httpsServerAccept();
|
||||
$self->httpsServerResponse(500);
|
||||
});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oS3Request->request(HTTP_VERB_GET)}, undef, 'successful request after retries');
|
||||
$self->testException(
|
||||
sub {$oS3Request->request(HTTP_VERB_GET)}, ERROR_PROTOCOL, 'S3 request error \[500\] GenericMessage.*');
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
Loading…
x
Reference in New Issue
Block a user