db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-06-02 22:57:34 +02:00
Code Issues Releases Activity

Various improvements to validation of backup and restore.

Browse Source
This commit is contained in:
David Steele 2016-12-10 09:11:12 -05:00
parent b45e0d8189
commit eff7b46eb1
8 changed files with 95 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -287,6 +287,10 @@
<release-item>
<p>Changed the <br-option>--no-fork</br-option> test option to <br-option>--fork</br-option> with negation to match all other boolean parameters.</p>
</release-item>
<release-item>
<p>Various improvements to validation of backup and restore.</p>
</release-item>
</release-refactor-list>
</release-test-list>
</release>

4
test/lib/pgBackRestTest/Backup/BackupTest.pm
View File

@ -779,6 +779,7 @@ sub backupTestRun
my %oManifest;
$oManifest{&INI_SECTION_BACKREST}{&INI_KEY_VERSION} = BACKREST_VERSION;
$oManifest{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = BACKREST_FORMAT;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY} = JSON::PP::false;
@ -1715,6 +1716,9 @@ sub backupTestRun
OPTION_DEFAULT_RESTORE_SET, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'no tablespace remap', undef, '--tablespace-map-all=../../tablespace --log-level-console=detail', false);
$oManifest{&MANIFEST_SECTION_BACKUP_TARGET}{'pg_tblspc/2'}{&MANIFEST_SUBKEY_PATH} = '../../tablespace/ts2';
$oManifest{&MANIFEST_SECTION_TARGET_LINK}{'pg_data/pg_tblspc/2'}{&MANIFEST_SUBKEY_DESTINATION} = '../../tablespace/ts2';
# Backup Info (with an empty stanza)
#-----------------------------------------------------------------------------------------------------------------------
executeTest('sudo chmod g+w ' . $oHostBackup->repoPath() . '/backup');

2
test/lib/pgBackRestTest/Backup/Common/ExpireCommonTest.pm
View File

@ -21,7 +21,7 @@ use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Version;
use pgBackRestTest::Backup::Common::HostBackupTest;
use pgBackRestTest::Backup::Common::HostBaseTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::HostGroupTest;

88
test/lib/pgBackRestTest/Backup/Common/HostBackupTest.pm
View File

@ -42,8 +42,6 @@ use constant HOST_BACKUP_USER => 'backup-u
####################################################################################################################################
use constant HOST_PARAM_BACKREST_CONFIG => 'backrest-config';
push @EXPORT, qw(HOST_PARAM_BACKREST_CONFIG);
use constant HOST_PARAM_BACKREST_EXE => 'backrest-exe';
push @EXPORT, qw(HOST_PARAM_BACKREST_EXE);
use constant HOST_PARAM_LOCK_PATH => 'lock-path';
push @EXPORT, qw(HOST_PARAM_LOCK_PATH);
use constant HOST_PARAM_LOG_PATH => 'log-path';
@ -260,7 +258,7 @@ sub backupEnd
);
# Set defaults
my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? $$oParam{oExpectedManifest} : undef;
my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? dclone($$oParam{oExpectedManifest}) : undef;
my $iExitStatus = $oExecuteBackup->end();
@ -472,7 +470,7 @@ sub backupCompare
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup;
my $oActualManifest = new pgBackRest::Common::Ini(
my $oActualManifest = new pgBackRest::Manifest(
$self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST));
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} =
@ -485,20 +483,50 @@ sub backupCompare
$oActualManifest->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM);
${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = BACKREST_FORMAT + 0;
foreach my $strPathKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_PATH))
{
my $strFileSection = MANIFEST_SECTION_TARGET_FILE;
my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH);
foreach my $strFileKey ($oActualManifest->keys($strFileSection))
foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE))
{
# Determine repo size if compression is enabled
if ($oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS})
{
if ($oActualManifest->test($strFileSection, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE))
my $lRepoSize =
$oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
$oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
(fileStat($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/${strFileKey}.gz")))->size;
if (defined($lRepoSize) &&
$lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
{
${$oExpectedManifest}{$strFileSection}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} =
$oActualManifest->get($strFileSection, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE);
$oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} = $lRepoSize;
}
}
}
$self->manifestDefault($oExpectedManifest);
my $strTestPath = $self->testPath();
iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
iniSave("${strTestPath}/expected.manifest", $oExpectedManifest);
executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
fileRemove("${strTestPath}/expected.manifest");
fileRemove("${strTestPath}/actual.manifest");
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# manifestDefault
####################################################################################################################################
sub manifestDefault
{
my $self = shift;
my $oExpectedManifest = shift;
# Set defaults for subkeys that tend to repeat
foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK)
{
@ -507,16 +535,11 @@ sub backupCompare
my %oDefault;
my $iSectionTotal = 0;
foreach my $strFile (keys(%{${$oExpectedManifest}{$strSection}}))
{
if (!defined(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey}) &&
defined(${$oExpectedManifest}{"${strSection}:default"}{$strSubKey}))
{
${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey} =
${$oExpectedManifest}{"${strSection}:default"}{$strSubKey};
}
next if !defined($oExpectedManifest->{$strSection});
my $strValue = ${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey};
foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}}))
{
my $strValue = $oExpectedManifest->{$strSection}{$strFile}{$strSubKey};
if (defined($strValue))
{
@ -549,37 +572,24 @@ sub backupCompare
{
if ($strSubKey eq MANIFEST_SUBKEY_MASTER)
{
${$oExpectedManifest}{"${strSection}:default"}{$strSubKey} = $strMaxValue ? JSON::PP::true : JSON::PP::false;
$oExpectedManifest->{"${strSection}:default"}{$strSubKey} = $strMaxValue ? JSON::PP::true : JSON::PP::false;
}
else
{
${$oExpectedManifest}{"${strSection}:default"}{$strSubKey} = $strMaxValue;
$oExpectedManifest->{"${strSection}:default"}{$strSubKey} = $strMaxValue;
}
foreach my $strFile (keys(%{${$oExpectedManifest}{$strSection}}))
foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}}))
{
if (defined(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey}) &&
${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey} eq $strMaxValue)
if (defined($oExpectedManifest->{$strSection}{$strFile}{$strSubKey}) &&
$oExpectedManifest->{$strSection}{$strFile}{$strSubKey} eq $strMaxValue)
{
delete(${$oExpectedManifest}{$strSection}{$strFile}{$strSubKey});
delete($oExpectedManifest->{$strSection}{$strFile}{$strSubKey});
}
}
}
}
}
my $strTestPath = $self->testPath();
iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
iniSave("${strTestPath}/expected.manifest", $oExpectedManifest);
executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
fileRemove("${strTestPath}/expected.manifest");
fileRemove("${strTestPath}/actual.manifest");
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################

2
test/lib/pgBackRestTest/Backup/Common/HostBaseTest.pm
View File

@ -38,6 +38,8 @@ use constant HOST_BACKUP => 'backup';
####################################################################################################################################
# Host parameters
####################################################################################################################################
use constant HOST_PARAM_BACKREST_EXE => 'backrest-exe';
push @EXPORT, qw(HOST_PARAM_BACKREST_EXE);
use constant HOST_PARAM_VM_ID => 'vm-id';
push @EXPORT, qw(HOST_PARAM_VM_ID);
use constant HOST_PARAM_TEST_PATH => 'test-path';

54
test/lib/pgBackRestTest/Backup/Common/HostDbCommonTest.pm
View File

@ -15,6 +15,7 @@ use DBI;
use Exporter qw(import);
our @EXPORT = qw();
use Fcntl ':mode';
use Storable qw(dclone);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
@ -391,7 +392,7 @@ sub restore
if (!defined($oExpectedManifestRef))
{
# Change mode on the backup path so it can be read
# Load the manifest
my $oExpectedManifest = new pgBackRest::Manifest(
$self->{oFile}->pathGet(
PATH_BACKUP_CLUSTER, ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST),
@ -471,7 +472,7 @@ sub restore
if (!defined($iExpectedExitStatus))
{
$self->restoreCompare($strBackup, $oExpectedManifestRef, $bTablespace);
$self->restoreCompare($strBackup, dclone($oExpectedManifestRef), $bTablespace);
if (defined($self->{oLogTest}))
{
@ -595,12 +596,9 @@ sub restoreCompare
${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{size});
}
if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE}))
{
$oActualManifest->numericSet(
MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE,
${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE});
}
# Remove repo-size from the manifest. ??? This could be improved to get actual sizes from the backup.
$oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE);
delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE});
if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0)
{
@ -654,21 +652,33 @@ sub restoreCompare
$oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef,
${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_VERSION});
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START});
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START});
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP});
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL});
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE});
$oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef,
${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM});
if (!$self->synthetic())
if ($self->synthetic())
{
$oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START);
$oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL);
$oActualManifest->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE);
}
else
{
$oActualManifest->set(
INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef, $oExpectedManifestRef->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM});
$oActualManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef,
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL});
$oActualManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START});
$oActualManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef,
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START});
$oActualManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef,
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP});
$oActualManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef,
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE});
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_START, undef,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_START});
$oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_STOP, undef,
@ -697,6 +707,8 @@ sub restoreCompare
# Delete the list of DBs
delete($$oExpectedManifestRef{&MANIFEST_SECTION_DB});
$self->manifestDefault($oExpectedManifestRef);
iniSave("${strTestPath}/actual.manifest", $oActualManifest->{oContent});
iniSave("${strTestPath}/expected.manifest", $oExpectedManifestRef);

1
test/lib/pgBackRestTest/Common/HostTest.pm
View File

@ -58,6 +58,7 @@ sub new
(defined($self->{stryMount}) ? ' -v ' . join(' -v ', @{$self->{stryMount}}) : '') .
" $self->{strImage}");
# Get IP Address
$self->{strIP} = trim(executeTest("docker inspect --format '\{\{ .NetworkSettings.IPAddress \}\}' $self->{strContainer}"));
$self->{bActive} = true;

2
test/lib/pgBackRestTest/Help/HelpTest.pm
View File

@ -16,7 +16,7 @@ use Exporter qw(import);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRestTest::Backup::Common::HostBackupTest;
use pgBackRestTest::Backup::Common::HostBaseTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::CommonTest;