Fix static WAL segment size used to determine if archive-push-queue-max has been exceeded.

This calculation was missed when the WAL segment size was made dynamic in preparation for PostgreSQL 11.

Fix the calculation by checking the actual WAL file sizes instead of using an estimate based on WAL segment size.  This is more accurate because it takes into account .history and .backup files, which are smaller.  Since the calculation is done in the async process the additional processing time should not adversely affect performance.

Remove the PG_WAL_SIZE constant and instead use local constants where the old value is still required.  This is only the case for some tests and PostgreSQL 8.3 which does not provide a way to get the WAL segment size from pg_control.

doc/xml/release.xml

@@ -23,6 +23,10 @@
                         <p>Fix issue with <br-option>archive-push-queue-max</br-option> not being honored on connection error.</p>
                     </release-item>
 
+                    <release-item>
+                        <p>Fix static WAL segment size used to determine if <br-option>archive-push-queue-max</br-option> has been exceeded.</p>
+                    </release-item>
+
                     <release-item>
                         <release-item-contributor-list>
                             <release-item-ideator id="vthriller"/>

lib/pgBackRest/Archive/Push/Push.pm

@@ -178,7 +178,7 @@ sub dropList
     my
     (
         $strOperation,
-        $stryReadyFile,
+        $stryWalFile,
     ) =
         logDebugParam
         (
@@ -186,12 +186,20 @@ sub dropList
             {name => 'stryReadyList'},
         );
 
+    # Determine the total size of the WAL
+    my $iTotalSize = 0;
+
+    for my $strWalFile (@{$stryWalFile})
+    {
+        $iTotalSize += (storageDb()->info("$self->{strWalPath}/${strWalFile}"))->size();
+    }
+
+    # If total size exceeds queue max then drop all files
     my $stryDropFile = [];
 
-    # Determine if there are any to be dropped
+    if ($iTotalSize > cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX))
-    if (@{$stryReadyFile} > int(cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) / PG_WAL_SIZE))
     {
-        $stryDropFile = $stryReadyFile;
+        $stryDropFile = $stryWalFile;
     }
 
     return logDebugReturn

lib/pgBackRest/Db.pm

@@ -25,6 +25,13 @@ use pgBackRest::Protocol::Helper;
 use pgBackRest::Protocol::Storage::Helper;
 use pgBackRest::Version;
 
+####################################################################################################################################
+# PostgreSQL 8.3 WAL size
+#
+# WAL segment size in 8.3 cannot be determined from pg_control, so use this constant instead.
+####################################################################################################################################
+use constant PG_WAL_SIZE_83                                         => 16777216;
+
 ####################################################################################################################################
 # Backup advisory lock
 ####################################################################################################################################
@@ -682,7 +689,7 @@ sub backupStart
 
     my ($strTimestampDbStart, $strArchiveStart, $strLsnStart, $iWalSegmentSize) = $self->executeSqlRow(
         "select to_char(current_timestamp, 'YYYY-MM-DD HH24:MI:SS.US TZ'), pg_" . $self->walId() . "file_name(lsn), lsn::text," .
-            ($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE :
+            ($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE_83 :
                 " (select setting::int8 from pg_settings where name = 'wal_segment_size')" .
                 # In Pre-11 versions the wal_segment_sise was expressed in terms of blocks rather than total size
                 ($self->{strDbVersion} < PG_VERSION_11 ?

lib/pgBackRest/DbVersion.pm

@@ -18,12 +18,6 @@ use pgBackRest::Common::Log;
 use constant PG_PAGE_SIZE                                           => 8192;
     push @EXPORT, qw(PG_PAGE_SIZE);
 
-####################################################################################################################################
-# Supported WAL size
-####################################################################################################################################
-use constant PG_WAL_SIZE                                            => 16777216;
-    push @EXPORT, qw(PG_WAL_SIZE);
-
 ####################################################################################################################################
 # PostgreSQL version numbers
 ####################################################################################################################################

src/perl/embed.auto.c

@@ -1957,19 +1957,26 @@ static const EmbeddedModule embeddedModule[] =
             "my\n"
             "(\n"
             "$strOperation,\n"
-            "$stryReadyFile,\n"
+            "$stryWalFile,\n"
             ") =\n"
             "logDebugParam\n"
             "(\n"
             "__PACKAGE__ . '->dropList', \\@_,\n"
             "{name => 'stryReadyList'},\n"
             ");\n"
-            "\n"
-            "my $stryDropFile = [];\n"
             "\n\n"
-            "if (@{$stryReadyFile} > int(cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) / PG_WAL_SIZE))\n"
+            "my $iTotalSize = 0;\n"
+            "\n"
+            "for my $strWalFile (@{$stryWalFile})\n"
             "{\n"
-            "$stryDropFile = $stryReadyFile;\n"
+            "$iTotalSize += (storageDb()->info(\"$self->{strWalPath}/${strWalFile}\"))->size();\n"
+            "}\n"
+            "\n\n"
+            "my $stryDropFile = [];\n"
+            "\n"
+            "if ($iTotalSize > cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX))\n"
+            "{\n"
+            "$stryDropFile = $stryWalFile;\n"
             "}\n"
             "\n"
             "return logDebugReturn\n"
@@ -8789,6 +8796,8 @@ static const EmbeddedModule embeddedModule[] =
             "use pgBackRest::Protocol::Helper;\n"
             "use pgBackRest::Protocol::Storage::Helper;\n"
             "use pgBackRest::Version;\n"
+            "\n\n\n\n\n\n"
+            "use constant PG_WAL_SIZE_83 => 16777216;\n"
             "\n\n\n\n"
             "use constant DB_BACKUP_ADVISORY_LOCK => '12340078987004321';\n"
             "push @EXPORT, qw(DB_BACKUP_ADVISORY_LOCK);\n"
@@ -9337,7 +9346,7 @@ static const EmbeddedModule embeddedModule[] =
             "\n"
             "my ($strTimestampDbStart, $strArchiveStart, $strLsnStart, $iWalSegmentSize) = $self->executeSqlRow(\n"
             "\"select to_char(current_timestamp, 'YYYY-MM-DD HH24:MI:SS.US TZ'), pg_\" . $self->walId() . \"file_name(lsn), lsn::text,\" .\n"
-            "($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE :\n"
+            "($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE_83 :\n"
             "\" (select setting::int8 from pg_settings where name = 'wal_segment_size')\" .\n"
             "\n"
             "($self->{strDbVersion} < PG_VERSION_11 ?\n"
@@ -9767,9 +9776,6 @@ static const EmbeddedModule embeddedModule[] =
             "use constant PG_PAGE_SIZE => 8192;\n"
             "push @EXPORT, qw(PG_PAGE_SIZE);\n"
             "\n\n\n\n"
-            "use constant PG_WAL_SIZE => 16777216;\n"
-            "push @EXPORT, qw(PG_WAL_SIZE);\n"
-            "\n\n\n\n"
             "use constant PG_VERSION_83 => '8.3';\n"
             "push @EXPORT, qw(PG_VERSION_83);\n"
             "use constant PG_VERSION_84 => '8.4';\n"

test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm

@@ -34,6 +34,11 @@ use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::Common::HostGroupTest;
 use pgBackRestTest::Common::RunTest;
 
+####################################################################################################################################
+# Test WAL size
+####################################################################################################################################
+use constant PG_WAL_SIZE_TEST                                       => 16777216;
+
 ####################################################################################################################################
 # Host defaults
 ####################################################################################################################################
@@ -145,7 +150,7 @@ sub archivePush
     $self->executeSimple(
         $self->backrestExe() .
         ' --config=' . $self->backrestConfig() .
-        ' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE) .
+        ' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE_TEST) .
         ' --stanza=' . $self->stanza() .
         (defined($iExpectedError) && $iExpectedError == ERROR_FILE_READ ? ' --repo1-host=bogus' : '') .
         ($bAsync ? '' : ' --no-archive-async') .

test/lib/pgBackRestTest/Module/Archive/ArchivePushPerlTest.pm

@@ -34,6 +34,11 @@ use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::Env::Host::HostBackupTest;
 use pgBackRestTest::Common::RunTest;
 
+####################################################################################################################################
+# Test WAL size
+####################################################################################################################################
+use constant PG_WAL_SIZE_TEST                                       => 16777216;
+
 ####################################################################################################################################
 # initModule
 ####################################################################################################################################
@@ -294,7 +299,7 @@ sub run
     if ($self->begin("ArchivePush->dropList()"))
     {
         my $oPushAsync = new pgBackRest::Archive::Push::Async($self->{strWalPath}, $self->{strSpoolPath});
-        $self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE * 4);
+        $self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 4);
         $self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
 
         my $iWalTimeline = 1;
@@ -302,16 +307,24 @@ sub run
         my $iWalMinor = 1;
 
         #---------------------------------------------------------------------------------------------------------------------------
-        storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
+        my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
-        storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
+        storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
-        storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
+        $self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
+
+        $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
+        storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
+        $self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
+
+        $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
+        storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
+        $self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
 
         $self->testResult(
             sub {$oPushAsync->dropList($oPushAsync->readyList())}, '()',
             'WAL files not dropped');
 
         #---------------------------------------------------------------------------------------------------------------------------
-        $self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE * 2);
+        $self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 2);
         $self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
 
         $self->testResult(
@@ -472,7 +485,7 @@ sub run
         $self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
 
         #---------------------------------------------------------------------------------------------------------------------------
-        $self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE * 2);
+        $self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 2);
         $self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
 
         # Generate WAL to test queue limits
@@ -570,7 +583,7 @@ sub run
             "${strSegment} WAL in archive");
 
         # Set more realistic queue max and allow segment to push
-        $self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE * 4);
+        $self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 4);
         $self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
 
         $self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, undef, "${strSegment} WAL pushed");