Various improvements to validation of backup and restore.

doc/xml/release.xml

@@ -287,6 +287,10 @@
                     <release-item>
                         <p>Changed the <br-option>--no-fork</br-option> test option to <br-option>--fork</br-option> with negation to match all other boolean parameters.</p>
                     </release-item>
+
+                    <release-item>
+                        <p>Various improvements to validation of backup and restore.</p>
+                    </release-item>
                 </release-refactor-list>
             </release-test-list>
         </release>

test/lib/pgBackRestTest/Backup/BackupTest.pm

@@ -779,6 +779,7 @@ sub backupTestRun
             my %oManifest;
 
             $oManifest{&INI_SECTION_BACKREST}{&INI_KEY_VERSION} = BACKREST_VERSION;
+            $oManifest{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = BACKREST_FORMAT;
             $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK} = JSON::PP::true;
             $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY} = JSON::PP::true;
             $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY} = JSON::PP::false;
@@ -1715,6 +1716,9 @@ sub backupTestRun
                 OPTION_DEFAULT_RESTORE_SET, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
                 'no tablespace remap', undef, '--tablespace-map-all=../../tablespace --log-level-console=detail', false);
 
+            $oManifest{&MANIFEST_SECTION_BACKUP_TARGET}{'pg_tblspc/2'}{&MANIFEST_SUBKEY_PATH} = '../../tablespace/ts2';
+            $oManifest{&MANIFEST_SECTION_TARGET_LINK}{'pg_data/pg_tblspc/2'}{&MANIFEST_SUBKEY_DESTINATION} = '../../tablespace/ts2';
+
             # Backup Info (with an empty stanza)
             #-----------------------------------------------------------------------------------------------------------------------
             executeTest('sudo chmod g+w ' . $oHostBackup->repoPath() . '/backup');

test/lib/pgBackRestTest/Backup/Common/ExpireCommonTest.pm

@@ -21,7 +21,7 @@ use pgBackRest::FileCommon;
 use pgBackRest::Manifest;
 use pgBackRest::Version;
 
-use pgBackRestTest::Backup::Common::HostBackupTest;
+use pgBackRestTest::Backup::Common::HostBaseTest;
 use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::Common::FileTest;
 use pgBackRestTest::Common::HostGroupTest;

test/lib/pgBackRestTest/Backup/Common/HostBackupTest.pm

@@ -42,8 +42,6 @@ use constant HOST_BACKUP_USER                                       => 'backup-u
 ####################################################################################################################################
 use constant HOST_PARAM_BACKREST_CONFIG                             => 'backrest-config';
     push @EXPORT, qw(HOST_PARAM_BACKREST_CONFIG);
-use constant HOST_PARAM_BACKREST_EXE                                => 'backrest-exe';
-    push @EXPORT, qw(HOST_PARAM_BACKREST_EXE);
 use constant HOST_PARAM_LOCK_PATH                                   => 'lock-path';
     push @EXPORT, qw(HOST_PARAM_LOCK_PATH);
 use constant HOST_PARAM_LOG_PATH                                    => 'log-path';
@@ -260,7 +258,7 @@ sub backupEnd
         );
 
     # Set defaults
-    my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? $$oParam{oExpectedManifest} : undef;
+    my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? dclone($$oParam{oExpectedManifest}) : undef;
 
     my $iExitStatus = $oExecuteBackup->end();
 
@@ -472,7 +470,7 @@ sub backupCompare
 
     ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup;
 
-    my $oActualManifest = new pgBackRest::Common::Ini(
+    my $oActualManifest = new pgBackRest::Manifest(
         $self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST));
 
     ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} =
@@ -485,20 +483,50 @@ sub backupCompare
         $oActualManifest->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM);
     ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = BACKREST_FORMAT + 0;
 
-    foreach my $strPathKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_PATH))
+    my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH);
-    {
-        my $strFileSection = MANIFEST_SECTION_TARGET_FILE;
 
-        foreach my $strFileKey ($oActualManifest->keys($strFileSection))
+    foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE))
+    {
+        # Determine repo size if compression is enabled
+        if ($oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS})
         {
-            if ($oActualManifest->test($strFileSection, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE))
+            my $lRepoSize =
+                $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
+                    $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
+                    (fileStat($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/${strFileKey}.gz")))->size;
+
+            if (defined($lRepoSize) &&
+                $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
             {
-                ${$oExpectedManifest}{$strFileSection}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} =
+                $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} = $lRepoSize;
-                    $oActualManifest->get($strFileSection, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE);
             }
         }
     }
 
+    $self->manifestDefault($oExpectedManifest);
+
+    my $strTestPath = $self->testPath();
+
+    iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
+    iniSave("${strTestPath}/expected.manifest", $oExpectedManifest);
+
+    executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
+
+    fileRemove("${strTestPath}/expected.manifest");
+    fileRemove("${strTestPath}/actual.manifest");
+
+    # Return from function and log return values if any
+    return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# manifestDefault
+####################################################################################################################################
+sub manifestDefault
+{
+    my $self = shift;
+    my $oExpectedManifest = shift;
+
     # Set defaults for subkeys that tend to repeat
     foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK)
     {
@@ -507,16 +535,11 @@ sub backupCompare
             my %oDefault;
             my $iSectionTotal = 0;
 
-            foreach my $strFile (keys(%{${$oExpectedManifest}{$strSection}}))
+            next if !defined($oExpectedManifest->{$strSection});
-            {
-                if (!defined(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey}) &&
-                    defined(${$oExpectedManifest}{"${strSection}:default"}{$strSubKey}))
-                {
-                    ${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey} =
-                        ${$oExpectedManifest}{"${strSection}:default"}{$strSubKey};
-                }
 
-                my $strValue = ${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey};
+            foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}}))
+            {
+                my $strValue = $oExpectedManifest->{$strSection}{$strFile}{$strSubKey};
 
                 if (defined($strValue))
                 {
@@ -549,37 +572,24 @@ sub backupCompare
             {
                 if ($strSubKey eq MANIFEST_SUBKEY_MASTER)
                 {
-                    ${$oExpectedManifest}{"${strSection}:default"}{$strSubKey} = $strMaxValue ? JSON::PP::true : JSON::PP::false;
+                    $oExpectedManifest->{"${strSection}:default"}{$strSubKey} = $strMaxValue ? JSON::PP::true : JSON::PP::false;
                 }
                 else
                 {
-                    ${$oExpectedManifest}{"${strSection}:default"}{$strSubKey} = $strMaxValue;
+                    $oExpectedManifest->{"${strSection}:default"}{$strSubKey} = $strMaxValue;
                 }
 
-                foreach my $strFile (keys(%{${$oExpectedManifest}{$strSection}}))
+                foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}}))
                 {
-                    if (defined(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey}) &&
+                    if (defined($oExpectedManifest->{$strSection}{$strFile}{$strSubKey}) &&
-                        ${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey} eq $strMaxValue)
+                        $oExpectedManifest->{$strSection}{$strFile}{$strSubKey} eq $strMaxValue)
                     {
-                        delete(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey});
+                        delete($oExpectedManifest->{$strSection}{$strFile}{$strSubKey});
                     }
                 }
             }
         }
     }
-
-    my $strTestPath = $self->testPath();
-
-    iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
-    iniSave("${strTestPath}/expected.manifest", $oExpectedManifest);
-
-    executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
-
-    fileRemove("${strTestPath}/expected.manifest");
-    fileRemove("${strTestPath}/actual.manifest");
-
-    # Return from function and log return values if any
-    return logDebugReturn($strOperation);
 }
 
 ####################################################################################################################################

test/lib/pgBackRestTest/Backup/Common/HostBaseTest.pm

@@ -38,6 +38,8 @@ use constant HOST_BACKUP                                            => 'backup';
 ####################################################################################################################################
 # Host parameters
 ####################################################################################################################################
+use constant HOST_PARAM_BACKREST_EXE                                => 'backrest-exe';
+    push @EXPORT, qw(HOST_PARAM_BACKREST_EXE);
 use constant HOST_PARAM_VM_ID                                       => 'vm-id';
     push @EXPORT, qw(HOST_PARAM_VM_ID);
 use constant HOST_PARAM_TEST_PATH                                   => 'test-path';

test/lib/pgBackRestTest/Backup/Common/HostDbCommonTest.pm

@@ -15,6 +15,7 @@ use DBI;
 use Exporter qw(import);
     our @EXPORT = qw();
 use Fcntl ':mode';
+use Storable qw(dclone);
 
 use pgBackRest::Common::Exception;
 use pgBackRest::Common::Ini;
@@ -391,7 +392,7 @@ sub restore
 
     if (!defined($oExpectedManifestRef))
     {
-        # Change mode on the backup path so it can be read
+        # Load the manifest
         my $oExpectedManifest = new pgBackRest::Manifest(
             $self->{oFile}->pathGet(
                 PATH_BACKUP_CLUSTER, ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST),
@@ -471,7 +472,7 @@ sub restore
 
     if (!defined($iExpectedExitStatus))
     {
-        $self->restoreCompare($strBackup, $oExpectedManifestRef, $bTablespace);
+        $self->restoreCompare($strBackup, dclone($oExpectedManifestRef), $bTablespace);
 
         if (defined($self->{oLogTest}))
         {
@@ -595,12 +596,9 @@ sub restoreCompare
                 ${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{size});
         }
 
-        if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE}))
+        # Remove repo-size from the manifest.  ??? This could be improved to get actual sizes from the backup.
-        {
+        $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE);
-            $oActualManifest->numericSet(
+        delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE});
-                MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE,
-                ${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE});
-        }
 
         if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0)
         {
@@ -654,21 +652,33 @@ sub restoreCompare
 
     $oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef,
                           ${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_VERSION});
-    $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
-                          ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START});
-    $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef,
-                          ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START});
-    $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef,
-                          ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP});
-    $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef,
-                          ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL});
-    $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef,
-                          ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE});
-    $oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef,
-                          ${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM});
 
-    if (!$self->synthetic())
+    if ($self->synthetic())
     {
+        $oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START);
+        $oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL);
+        $oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE);
+    }
+    else
+    {
+        $oActualManifest->set(
+            INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef, $oExpectedManifestRef->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM});
+        $oActualManifest->set(
+            MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef,
+            $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL});
+        $oActualManifest->set(
+            MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
+            $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START});
+        $oActualManifest->set(
+            MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef,
+            $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START});
+        $oActualManifest->set(
+            MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef,
+            $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP});
+        $oActualManifest->set(
+            MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef,
+            $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE});
+
         $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_START, undef,
                               ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_START});
         $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_STOP, undef,
@@ -697,6 +707,8 @@ sub restoreCompare
     # Delete the list of DBs
     delete($$oExpectedManifestRef{&MANIFEST_SECTION_DB});
 
+    $self->manifestDefault($oExpectedManifestRef);
+
     iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
     iniSave("${strTestPath}/expected.manifest", $oExpectedManifestRef);
 

test/lib/pgBackRestTest/Common/HostTest.pm

@@ -58,6 +58,7 @@ sub new
                 (defined($self->{stryMount}) ? ' -v ' . join(' -v ', @{$self->{stryMount}}) : '') .
                 " $self->{strImage}");
 
+    # Get IP Address
     $self->{strIP} = trim(executeTest("docker inspect --format '\{\{ .NetworkSettings.IPAddress \}\}' $self->{strContainer}"));
     $self->{bActive} = true;
 

test/lib/pgBackRestTest/Help/HelpTest.pm

@@ -16,7 +16,7 @@ use Exporter qw(import);
 use pgBackRest::Common::Log;
 use pgBackRest::Config::Config;
 
-use pgBackRestTest::Backup::Common::HostBackupTest;
+use pgBackRestTest::Backup::Common::HostBaseTest;
 use pgBackRestTest::Common::HostGroupTest;
 use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::CommonTest;