Fixed an issue that could cause copying large manifests to fail during restore.

Reported by Craig A. James.

doc/xml/release.xml

@@ -12,6 +12,16 @@
     <release-list>
         <release date="XXXX-XX-XX" version="1.26dev" title="UNDER DEVELOPMENT">
             <release-core-list>
+                <release-bug-list>
+                    <release-item>
+                        <release-item-contributor-list>
+                            <release-item-ideator id="james.craig"/>
+                        </release-item-contributor-list>
+
+                        <p>Fixed an issue that could cause copying large manifests to fail during restore.</p>
+                    </release-item>
+                </release-bug-list>
+
                 <release-feature-list>
                     <release-item>
                         <release-item-contributor-list>
@@ -3566,6 +3576,11 @@
             <contributor-id type="github">crunchyjohn</contributor-id>
         </contributor>
 
+        <contributor id="james.craig">
+            <contributor-name-display>Craig A. James</contributor-name-display>
+            <contributor-id type="github">cjames53</contributor-id>
+        </contributor>
+
         <contributor id="kulkarni.nikhilchandra">
             <contributor-name-display>Nikhilchandra Kulkarni</contributor-name-display>
             <contributor-id type="github">nikhilchandra-kulkarni</contributor-id>

lib/pgBackRest/Protocol/Storage/File.pm

@@ -47,6 +47,9 @@ sub new
     # Set read/write
     $self->{bWrite} = false;
 
+    # Initialize EOF to false
+    $self->eofSet(false);
+
     # Return from function and log return values if any
     return logDebugReturn
     (
@@ -55,6 +58,25 @@ sub new
     );
 }
 
+####################################################################################################################################
+# eof - have reads reached eof?
+####################################################################################################################################
+sub eof
+{
+    return shift->{bEOF};
+}
+
+####################################################################################################################################
+# eofSet - set eof
+####################################################################################################################################
+sub eofSet
+{
+    my $self = shift;
+    my $bEOF = shift;
+
+    $self->{bEOF} = $bEOF;
+}
+
 ####################################################################################################################################
 # read - read block from protocol
 ####################################################################################################################################
@@ -63,6 +85,9 @@ sub read
     my $self = shift;
     my $rtBuffer = shift;
 
+    # After EOF always return 0
+    return 0 if $self->eof();
+
     my $lBlockSize;
 
     # Read the block header and make sure it's valid
@@ -81,6 +106,10 @@ sub read
     {
         $self->{oProtocol}->io()->read($rtBuffer, $lBlockSize, true);
     }
+    else
+    {
+        $self->eofSet(true);
+    }
 
     # Return the block size
     return $lBlockSize;

test/expect/real-all-001.log

@@ -159,12 +159,12 @@ restore, type 'default', expect exit 40 - path not empty (db-master host)
 ------------------------------------------------------------------------------------------------------------------------------------
 
 restore, type 'default' (db-master host)
-> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf  --db-include=test1  --link-all --stanza=db restore
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf  --db-include=test1 --buffer-size=16384  --link-all --stanza=db restore
 ------------------------------------------------------------------------------------------------------------------------------------
 
 + supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
 ----------------------------------------------------------------
-restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+restore_command = '[BACKREST-BIN] --buffer-size=16384 --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
 
 restore, force, backup '[BACKUP-FULL-1]', type 'immediate' (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --set=[BACKUP-FULL-1] --type=immediate --link-all --stanza=db restore

test/expect/real-all-002.log

@@ -213,12 +213,12 @@ protocol-timeout=60
 spool-path=[TEST_PATH]/db-standby/spool
 
 restore, type 'default' (db-master host)
-> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf  --buffer-size=16384  --link-all --stanza=db restore
 ------------------------------------------------------------------------------------------------------------------------------------
 
 + supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
 ----------------------------------------------------------------
-restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+restore_command = '[BACKREST-BIN] --buffer-size=16384 --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
 
 restore, force, backup '[BACKUP-FULL-1]', type 'immediate' (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --set=[BACKUP-FULL-1] --type=immediate --link-all --stanza=db restore

test/expect/real-all-003.log

@@ -219,12 +219,12 @@ archive-copy=y
 start-fast=y
 
 restore, type 'default' (db-master host)
-> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf  --buffer-size=16384  --link-all --stanza=db restore
 ------------------------------------------------------------------------------------------------------------------------------------
 
 + supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
 ----------------------------------------------------------------
-restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+restore_command = '[BACKREST-BIN] --buffer-size=16384 --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
 
 restore, force, backup '[BACKUP-FULL-1]', type 'immediate' (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --set=[BACKUP-FULL-1] --type=immediate --link-all --stanza=db restore

test/expect/real-all-004.log

@@ -217,12 +217,12 @@ archive-copy=y
 start-fast=y
 
 restore, type 'default' (db-master host)
-> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf  --buffer-size=16384  --link-all --stanza=db restore
 ------------------------------------------------------------------------------------------------------------------------------------
 
 + supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
 ----------------------------------------------------------------
-restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+restore_command = '[BACKREST-BIN] --buffer-size=16384 --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
 
 restore delta, backup '[BACKUP-FULL-1]', type 'time', target '[TIMESTAMP-TARGET-1]' (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-FULL-1] --type=time --target="[TIMESTAMP-TARGET-1]" --link-all --stanza=db restore

test/expect/real-all-005.log

@@ -300,12 +300,12 @@ archive-copy=y
 start-fast=y
 
 restore, type 'default' (db-master host)
-> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf  --buffer-size=16384  --link-all --stanza=db restore
 ------------------------------------------------------------------------------------------------------------------------------------
 
 + supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
 ----------------------------------------------------------------
-restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+restore_command = '[BACKREST-BIN] --buffer-size=16384 --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
 
 restore, force, backup '[BACKUP-FULL-1]', type 'immediate' (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --set=[BACKUP-FULL-1] --type=immediate --link-all --stanza=db restore

test/expect/real-all-006.log

@@ -223,12 +223,12 @@ archive-copy=y
 start-fast=y
 
 restore, type 'default' (db-master host)
-> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf  --buffer-size=16384  --link-all --stanza=db restore
 ------------------------------------------------------------------------------------------------------------------------------------
 
 + supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
 ----------------------------------------------------------------
-restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+restore_command = '[BACKREST-BIN] --buffer-size=16384 --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
 
 restore delta, backup '[BACKUP-FULL-1]', type 'time', target '[TIMESTAMP-TARGET-1]' (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-FULL-1] --type=time --target="[TIMESTAMP-TARGET-1]" --link-all --stanza=db restore

test/lib/pgBackRestTest/Module/Real/RealAllTest.pm

@@ -729,7 +729,7 @@ sub run
         $oHostDbMaster->restore(
             cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), undef, undef, $bDelta, $bForce, $strType, $strTarget,
             $bTargetExclusive, $strTargetAction, $strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus,
-            $bTestLocal ? ' --db-include=test1' : undef);
+            ($bTestLocal ? ' --db-include=test1' : '') . ' --buffer-size=16384');
 
         $oHostDbMaster->clusterStart();
         $oHostDbMaster->sqlSelectOneTest('select message from test', $bTestLocal ? $strNameMessage : $strIncrMessage);