db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-18 04:58:51 +02:00
Code Issues Releases Activity

Fix static WAL segment size used to determine if archive-push-queue-max has been exceeded.

Browse Source 
This calculation was missed when the WAL segment size was made dynamic in preparation for PostgreSQL 11.

Fix the calculation by checking the actual WAL file sizes instead of using an estimate based on WAL segment size.  This is more accurate because it takes into account .history and .backup files, which are smaller.  Since the calculation is done in the async process the additional processing time should not adversely affect performance.

Remove the PG_WAL_SIZE constant and instead use local constants where the old value is still required.  This is only the case for some tests and PostgreSQL 8.3 which does not provide a way to get the WAL segment size from pg_control.
This commit is contained in:
David Steele 2018-10-27 20:00:00 +01:00
parent 41b00dc204
commit 286f7e5011
7 changed files with 65 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -23,6 +23,10 @@
<p>Fix issue with <br-option>archive-push-queue-max</br-option> not being honored on connection error.</p>
</release-item>
<release-item>
<p>Fix static WAL segment size used to determine if <br-option>archive-push-queue-max</br-option> has been exceeded.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="vthriller"/>

16
lib/pgBackRest/Archive/Push/Push.pm
View File

@ -178,7 +178,7 @@ sub dropList
my
(
$strOperation,
$stryReadyFile,
$stryWalFile,
) =
logDebugParam
(
@ -186,12 +186,20 @@ sub dropList
{name => 'stryReadyList'},
);
# Determine the total size of the WAL
my $iTotalSize = 0;
for my $strWalFile (@{$stryWalFile})
{
$iTotalSize += (storageDb()->info("$self->{strWalPath}/${strWalFile}"))->size();
}
# If total size exceeds queue max then drop all files
my $stryDropFile = [];
# Determine if there are any to be dropped
if (@{$stryReadyFile} > int(cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) / PG_WAL_SIZE))
if ($iTotalSize > cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX))
{
$stryDropFile = $stryReadyFile;
$stryDropFile = $stryWalFile;
}
return logDebugReturn

9
lib/pgBackRest/Db.pm
View File

@ -25,6 +25,13 @@ use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
# PostgreSQL 8.3 WAL size
#
# WAL segment size in 8.3 cannot be determined from pg_control, so use this constant instead.
####################################################################################################################################
use constant PG_WAL_SIZE_83 => 16777216;
####################################################################################################################################
# Backup advisory lock
####################################################################################################################################
@ -682,7 +689,7 @@ sub backupStart
my ($strTimestampDbStart, $strArchiveStart, $strLsnStart, $iWalSegmentSize) = $self->executeSqlRow(
"select to_char(current_timestamp, 'YYYY-MM-DD HH24:MI:SS.US TZ'), pg_" . $self->walId() . "file_name(lsn), lsn::text," .
($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE :
($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE_83 :
" (select setting::int8 from pg_settings where name = 'wal_segment_size')" .
# In Pre-11 versions the wal_segment_sise was expressed in terms of blocks rather than total size
($self->{strDbVersion} < PG_VERSION_11 ?

6
lib/pgBackRest/DbVersion.pm
View File

@ -18,12 +18,6 @@ use pgBackRest::Common::Log;
use constant PG_PAGE_SIZE => 8192;
push @EXPORT, qw(PG_PAGE_SIZE);
####################################################################################################################################
# Supported WAL size
####################################################################################################################################
use constant PG_WAL_SIZE => 16777216;
push @EXPORT, qw(PG_WAL_SIZE);
####################################################################################################################################
# PostgreSQL version numbers
####################################################################################################################################

24
src/perl/embed.auto.c
View File

@ -1957,19 +1957,26 @@ static const EmbeddedModule embeddedModule[] =
"my\n"
"(\n"
"$strOperation,\n"
"$stryReadyFile,\n"
"$stryWalFile,\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '->dropList', \\@_,\n"
"{name => 'stryReadyList'},\n"
");\n"
"\n"
"my $stryDropFile = [];\n"
"\n\n"
"if (@{$stryReadyFile} > int(cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) / PG_WAL_SIZE))\n"
"my $iTotalSize = 0;\n"
"\n"
"for my $strWalFile (@{$stryWalFile})\n"
"{\n"
"$stryDropFile = $stryReadyFile;\n"
"$iTotalSize += (storageDb()->info(\"$self->{strWalPath}/${strWalFile}\"))->size();\n"
"}\n"
"\n\n"
"my $stryDropFile = [];\n"
"\n"
"if ($iTotalSize > cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX))\n"
"{\n"
"$stryDropFile = $stryWalFile;\n"
"}\n"
"\n"
"return logDebugReturn\n"
@ -8789,6 +8796,8 @@ static const EmbeddedModule embeddedModule[] =
"use pgBackRest::Protocol::Helper;\n"
"use pgBackRest::Protocol::Storage::Helper;\n"
"use pgBackRest::Version;\n"
"\n\n\n\n\n\n"
"use constant PG_WAL_SIZE_83 => 16777216;\n"
"\n\n\n\n"
"use constant DB_BACKUP_ADVISORY_LOCK => '12340078987004321';\n"
"push @EXPORT, qw(DB_BACKUP_ADVISORY_LOCK);\n"
@ -9337,7 +9346,7 @@ static const EmbeddedModule embeddedModule[] =
"\n"
"my ($strTimestampDbStart, $strArchiveStart, $strLsnStart, $iWalSegmentSize) = $self->executeSqlRow(\n"
"\"select to_char(current_timestamp, 'YYYY-MM-DD HH24:MI:SS.US TZ'), pg_\" . $self->walId() . \"file_name(lsn), lsn::text,\" .\n"
"($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE :\n"
"($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE_83 :\n"
"\" (select setting::int8 from pg_settings where name = 'wal_segment_size')\" .\n"
"\n"
"($self->{strDbVersion} < PG_VERSION_11 ?\n"
@ -9767,9 +9776,6 @@ static const EmbeddedModule embeddedModule[] =
"use constant PG_PAGE_SIZE => 8192;\n"
"push @EXPORT, qw(PG_PAGE_SIZE);\n"
"\n\n\n\n"
"use constant PG_WAL_SIZE => 16777216;\n"
"push @EXPORT, qw(PG_WAL_SIZE);\n"
"\n\n\n\n"
"use constant PG_VERSION_83 => '8.3';\n"
"push @EXPORT, qw(PG_VERSION_83);\n"
"use constant PG_VERSION_84 => '8.4';\n"

7
test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm
View File

@ -34,6 +34,11 @@ use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# Test WAL size
####################################################################################################################################
use constant PG_WAL_SIZE_TEST => 16777216;
####################################################################################################################################
# Host defaults
####################################################################################################################################
@ -145,7 +150,7 @@ sub archivePush
$self->executeSimple(
$self->backrestExe() .
' --config=' . $self->backrestConfig() .
' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE) .
' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE_TEST) .
' --stanza=' . $self->stanza() .
(defined($iExpectedError) && $iExpectedError == ERROR_FILE_READ ? ' --repo1-host=bogus' : '') .
($bAsync ? '' : ' --no-archive-async') .

27
test/lib/pgBackRestTest/Module/Archive/ArchivePushPerlTest.pm
View File

@ -34,6 +34,11 @@ use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# Test WAL size
####################################################################################################################################
use constant PG_WAL_SIZE_TEST => 16777216;
####################################################################################################################################
# initModule
####################################################################################################################################
@ -294,7 +299,7 @@ sub run
if ($self->begin("ArchivePush->dropList()"))
{
my $oPushAsync = new pgBackRest::Archive::Push::Async($self->{strWalPath}, $self->{strSpoolPath});
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE * 4);
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 4);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
my $iWalTimeline = 1;
@ -302,16 +307,24 @@ sub run
my $iWalMinor = 1;
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$self->testResult(
sub {$oPushAsync->dropList($oPushAsync->readyList())}, '()',
'WAL files not dropped');
#---------------------------------------------------------------------------------------------------------------------------
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE * 2);
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 2);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testResult(
@ -472,7 +485,7 @@ sub run
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
#---------------------------------------------------------------------------------------------------------------------------
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE * 2);
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 2);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Generate WAL to test queue limits
@ -570,7 +583,7 @@ sub run
"${strSegment} WAL in archive");
# Set more realistic queue max and allow segment to push
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE * 4);
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 4);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, undef, "${strSegment} WAL pushed");