diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index 35760a224..d5dd2ceb2 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -287,6 +287,10 @@
Changed the --no-fork test option to --fork with negation to match all other boolean parameters.
+
+
+ Various improvements to validation of backup and restore.
+
diff --git a/test/lib/pgBackRestTest/Backup/BackupTest.pm b/test/lib/pgBackRestTest/Backup/BackupTest.pm
index a3c5016b9..7ac7a27f2 100755
--- a/test/lib/pgBackRestTest/Backup/BackupTest.pm
+++ b/test/lib/pgBackRestTest/Backup/BackupTest.pm
@@ -779,6 +779,7 @@ sub backupTestRun
my %oManifest;
$oManifest{&INI_SECTION_BACKREST}{&INI_KEY_VERSION} = BACKREST_VERSION;
+ $oManifest{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = BACKREST_FORMAT;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY} = JSON::PP::false;
@@ -1715,6 +1716,9 @@ sub backupTestRun
OPTION_DEFAULT_RESTORE_SET, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'no tablespace remap', undef, '--tablespace-map-all=../../tablespace --log-level-console=detail', false);
+ $oManifest{&MANIFEST_SECTION_BACKUP_TARGET}{'pg_tblspc/2'}{&MANIFEST_SUBKEY_PATH} = '../../tablespace/ts2';
+ $oManifest{&MANIFEST_SECTION_TARGET_LINK}{'pg_data/pg_tblspc/2'}{&MANIFEST_SUBKEY_DESTINATION} = '../../tablespace/ts2';
+
# Backup Info (with an empty stanza)
#-----------------------------------------------------------------------------------------------------------------------
executeTest('sudo chmod g+w ' . $oHostBackup->repoPath() . '/backup');
diff --git a/test/lib/pgBackRestTest/Backup/Common/ExpireCommonTest.pm b/test/lib/pgBackRestTest/Backup/Common/ExpireCommonTest.pm
index 14f79d899..0a69f1472 100644
--- a/test/lib/pgBackRestTest/Backup/Common/ExpireCommonTest.pm
+++ b/test/lib/pgBackRestTest/Backup/Common/ExpireCommonTest.pm
@@ -21,7 +21,7 @@ use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Version;
-use pgBackRestTest::Backup::Common::HostBackupTest;
+use pgBackRestTest::Backup::Common::HostBaseTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::HostGroupTest;
diff --git a/test/lib/pgBackRestTest/Backup/Common/HostBackupTest.pm b/test/lib/pgBackRestTest/Backup/Common/HostBackupTest.pm
index ae1c173e8..c2d1cb5d2 100644
--- a/test/lib/pgBackRestTest/Backup/Common/HostBackupTest.pm
+++ b/test/lib/pgBackRestTest/Backup/Common/HostBackupTest.pm
@@ -42,8 +42,6 @@ use constant HOST_BACKUP_USER => 'backup-u
####################################################################################################################################
use constant HOST_PARAM_BACKREST_CONFIG => 'backrest-config';
push @EXPORT, qw(HOST_PARAM_BACKREST_CONFIG);
-use constant HOST_PARAM_BACKREST_EXE => 'backrest-exe';
- push @EXPORT, qw(HOST_PARAM_BACKREST_EXE);
use constant HOST_PARAM_LOCK_PATH => 'lock-path';
push @EXPORT, qw(HOST_PARAM_LOCK_PATH);
use constant HOST_PARAM_LOG_PATH => 'log-path';
@@ -260,7 +258,7 @@ sub backupEnd
);
# Set defaults
- my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? $$oParam{oExpectedManifest} : undef;
+ my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? dclone($$oParam{oExpectedManifest}) : undef;
my $iExitStatus = $oExecuteBackup->end();
@@ -472,7 +470,7 @@ sub backupCompare
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup;
- my $oActualManifest = new pgBackRest::Common::Ini(
+ my $oActualManifest = new pgBackRest::Manifest(
$self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST));
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} =
@@ -485,20 +483,50 @@ sub backupCompare
$oActualManifest->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM);
${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = BACKREST_FORMAT + 0;
- foreach my $strPathKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_PATH))
- {
- my $strFileSection = MANIFEST_SECTION_TARGET_FILE;
+ my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH);
- foreach my $strFileKey ($oActualManifest->keys($strFileSection))
+ foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE))
+ {
+ # Determine repo size if compression is enabled
+ if ($oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS})
{
- if ($oActualManifest->test($strFileSection, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE))
+ my $lRepoSize =
+ $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
+ $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
+ (fileStat($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/${strFileKey}.gz")))->size;
+
+ if (defined($lRepoSize) &&
+ $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
{
- ${$oExpectedManifest}{$strFileSection}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} =
- $oActualManifest->get($strFileSection, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE);
+ $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} = $lRepoSize;
}
}
}
+ $self->manifestDefault($oExpectedManifest);
+
+ my $strTestPath = $self->testPath();
+
+ iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
+ iniSave("${strTestPath}/expected.manifest", $oExpectedManifest);
+
+ executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
+
+ fileRemove("${strTestPath}/expected.manifest");
+ fileRemove("${strTestPath}/actual.manifest");
+
+ # Return from function and log return values if any
+ return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# manifestDefault
+####################################################################################################################################
+sub manifestDefault
+{
+ my $self = shift;
+ my $oExpectedManifest = shift;
+
# Set defaults for subkeys that tend to repeat
foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK)
{
@@ -507,16 +535,11 @@ sub backupCompare
my %oDefault;
my $iSectionTotal = 0;
- foreach my $strFile (keys(%{${$oExpectedManifest}{$strSection}}))
- {
- if (!defined(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey}) &&
- defined(${$oExpectedManifest}{"${strSection}:default"}{$strSubKey}))
- {
- ${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey} =
- ${$oExpectedManifest}{"${strSection}:default"}{$strSubKey};
- }
+ next if !defined($oExpectedManifest->{$strSection});
- my $strValue = ${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey};
+ foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}}))
+ {
+ my $strValue = $oExpectedManifest->{$strSection}{$strFile}{$strSubKey};
if (defined($strValue))
{
@@ -549,37 +572,24 @@ sub backupCompare
{
if ($strSubKey eq MANIFEST_SUBKEY_MASTER)
{
- ${$oExpectedManifest}{"${strSection}:default"}{$strSubKey} = $strMaxValue ? JSON::PP::true : JSON::PP::false;
+ $oExpectedManifest->{"${strSection}:default"}{$strSubKey} = $strMaxValue ? JSON::PP::true : JSON::PP::false;
}
else
{
- ${$oExpectedManifest}{"${strSection}:default"}{$strSubKey} = $strMaxValue;
+ $oExpectedManifest->{"${strSection}:default"}{$strSubKey} = $strMaxValue;
}
- foreach my $strFile (keys(%{${$oExpectedManifest}{$strSection}}))
+ foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}}))
{
- if (defined(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey}) &&
- ${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey} eq $strMaxValue)
+ if (defined($oExpectedManifest->{$strSection}{$strFile}{$strSubKey}) &&
+ $oExpectedManifest->{$strSection}{$strFile}{$strSubKey} eq $strMaxValue)
{
- delete(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey});
+ delete($oExpectedManifest->{$strSection}{$strFile}{$strSubKey});
}
}
}
}
}
-
- my $strTestPath = $self->testPath();
-
- iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
- iniSave("${strTestPath}/expected.manifest", $oExpectedManifest);
-
- executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
-
- fileRemove("${strTestPath}/expected.manifest");
- fileRemove("${strTestPath}/actual.manifest");
-
- # Return from function and log return values if any
- return logDebugReturn($strOperation);
}
####################################################################################################################################
diff --git a/test/lib/pgBackRestTest/Backup/Common/HostBaseTest.pm b/test/lib/pgBackRestTest/Backup/Common/HostBaseTest.pm
index 1510dc781..8c905efa7 100644
--- a/test/lib/pgBackRestTest/Backup/Common/HostBaseTest.pm
+++ b/test/lib/pgBackRestTest/Backup/Common/HostBaseTest.pm
@@ -38,6 +38,8 @@ use constant HOST_BACKUP => 'backup';
####################################################################################################################################
# Host parameters
####################################################################################################################################
+use constant HOST_PARAM_BACKREST_EXE => 'backrest-exe';
+ push @EXPORT, qw(HOST_PARAM_BACKREST_EXE);
use constant HOST_PARAM_VM_ID => 'vm-id';
push @EXPORT, qw(HOST_PARAM_VM_ID);
use constant HOST_PARAM_TEST_PATH => 'test-path';
diff --git a/test/lib/pgBackRestTest/Backup/Common/HostDbCommonTest.pm b/test/lib/pgBackRestTest/Backup/Common/HostDbCommonTest.pm
index 529c7bbe5..22fb3e111 100644
--- a/test/lib/pgBackRestTest/Backup/Common/HostDbCommonTest.pm
+++ b/test/lib/pgBackRestTest/Backup/Common/HostDbCommonTest.pm
@@ -15,6 +15,7 @@ use DBI;
use Exporter qw(import);
our @EXPORT = qw();
use Fcntl ':mode';
+use Storable qw(dclone);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
@@ -391,7 +392,7 @@ sub restore
if (!defined($oExpectedManifestRef))
{
- # Change mode on the backup path so it can be read
+ # Load the manifest
my $oExpectedManifest = new pgBackRest::Manifest(
$self->{oFile}->pathGet(
PATH_BACKUP_CLUSTER, ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST),
@@ -471,7 +472,7 @@ sub restore
if (!defined($iExpectedExitStatus))
{
- $self->restoreCompare($strBackup, $oExpectedManifestRef, $bTablespace);
+ $self->restoreCompare($strBackup, dclone($oExpectedManifestRef), $bTablespace);
if (defined($self->{oLogTest}))
{
@@ -595,12 +596,9 @@ sub restoreCompare
${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{size});
}
- if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE}))
- {
- $oActualManifest->numericSet(
- MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE,
- ${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE});
- }
+ # Remove repo-size from the manifest. ??? This could be improved to get actual sizes from the backup.
+ $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE);
+ delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE});
if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0)
{
@@ -654,21 +652,33 @@ sub restoreCompare
$oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef,
${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_VERSION});
- $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
- ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START});
- $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef,
- ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START});
- $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef,
- ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP});
- $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef,
- ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL});
- $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef,
- ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE});
- $oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef,
- ${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM});
- if (!$self->synthetic())
+ if ($self->synthetic())
{
+ $oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START);
+ $oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL);
+ $oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE);
+ }
+ else
+ {
+ $oActualManifest->set(
+ INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef, $oExpectedManifestRef->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM});
+ $oActualManifest->set(
+ MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef,
+ $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL});
+ $oActualManifest->set(
+ MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
+ $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START});
+ $oActualManifest->set(
+ MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef,
+ $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START});
+ $oActualManifest->set(
+ MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef,
+ $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP});
+ $oActualManifest->set(
+ MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef,
+ $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE});
+
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_START, undef,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_START});
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_STOP, undef,
@@ -697,6 +707,8 @@ sub restoreCompare
# Delete the list of DBs
delete($$oExpectedManifestRef{&MANIFEST_SECTION_DB});
+ $self->manifestDefault($oExpectedManifestRef);
+
iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
iniSave("${strTestPath}/expected.manifest", $oExpectedManifestRef);
diff --git a/test/lib/pgBackRestTest/Common/HostTest.pm b/test/lib/pgBackRestTest/Common/HostTest.pm
index 215eee9c8..624cb0942 100644
--- a/test/lib/pgBackRestTest/Common/HostTest.pm
+++ b/test/lib/pgBackRestTest/Common/HostTest.pm
@@ -58,6 +58,7 @@ sub new
(defined($self->{stryMount}) ? ' -v ' . join(' -v ', @{$self->{stryMount}}) : '') .
" $self->{strImage}");
+ # Get IP Address
$self->{strIP} = trim(executeTest("docker inspect --format '\{\{ .NetworkSettings.IPAddress \}\}' $self->{strContainer}"));
$self->{bActive} = true;
diff --git a/test/lib/pgBackRestTest/Help/HelpTest.pm b/test/lib/pgBackRestTest/Help/HelpTest.pm
index 1bb16b9a7..e1e8563b4 100644
--- a/test/lib/pgBackRestTest/Help/HelpTest.pm
+++ b/test/lib/pgBackRestTest/Help/HelpTest.pm
@@ -16,7 +16,7 @@ use Exporter qw(import);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
-use pgBackRestTest::Backup::Common::HostBackupTest;
+use pgBackRestTest::Backup::Common::HostBaseTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::CommonTest;