1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
pgbackrest/test/lib/pgBackRestTest/Backup/BackupTest.pm
David Steele d18cd7ad0f Fixed an issue that prevented errors from being output to the console before the logging system was initialized.
That is, while parsing options. Error codes were still being returned accurately so this would not have made a process look like it succeeded when it did not.
2016-12-03 18:01:17 -05:00

2574 lines
129 KiB
Perl
Executable File

####################################################################################################################################
# BackupTest.pm - Unit Tests for pgBackRest::Backup and pgBackRest::Restore
####################################################################################################################################
package pgBackRestTest::Backup::BackupTest;
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
use File::Basename qw(dirname);
use pgBackRest::Archive;
use pgBackRest::ArchiveInfo;
use pgBackRest::BackupInfo;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Db;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Common;
use pgBackRest::Protocol::RemoteMaster;
use pgBackRest::Version;
use pgBackRestTest::Backup::BackupCommonTest;
use pgBackRestTest::Backup::Common::ExpireCommonTest;
use pgBackRestTest::Backup::Common::HostBackupTest;
use pgBackRestTest::Backup::Common::HostBaseTest;
use pgBackRestTest::Backup::Common::HostDbTest;
use pgBackRestTest::Backup::Common::HostDbSyntheticTest;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::HostTest;
use pgBackRestTest::Common::VmTest;
use pgBackRestTest::CommonTest;
####################################################################################################################################
# Constants
####################################################################################################################################
use constant WAL_VERSION_94 => '94';
####################################################################################################################################
# Archive helper functions
####################################################################################################################################
# Generate an archive log for testing
sub archiveGenerate
{
my $oFile = shift;
my $strXlogPath = shift;
my $iSourceNo = shift;
my $iArchiveNo = shift;
my $strWalVersion = shift;
my $bPartial = shift;
my $strArchiveFile = uc(sprintf('0000000100000001%08x', $iArchiveNo)) .
(defined($bPartial) && $bPartial ? '.partial' : '');
my $strArchiveTestFile = testDataPath() . "/backup.wal${iSourceNo}_${strWalVersion}.bin";
my $strSourceFile = "${strXlogPath}/${strArchiveFile}";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false); # Destination is not compressed
return $strArchiveFile, $strSourceFile;
}
# Check that an archive log is present
sub archiveCheck
{
my $oFile = shift;
my $strArchiveFile = shift;
my $strArchiveChecksum = shift;
my $bCompress = shift;
# Build the archive name to check for at the destination
my $strArchiveCheck = PG_VERSION_94 . "-1/${strArchiveFile}-${strArchiveChecksum}";
if ($bCompress)
{
$strArchiveCheck .= '.gz';
}
my $oWait = waitInit(5);
my $bFound = false;
do
{
$bFound = $oFile->exists(PATH_BACKUP_ARCHIVE, $strArchiveCheck);
}
while (!$bFound && waitMore($oWait));
if (!$bFound)
{
confess 'unable to find ' . $strArchiveCheck;
}
}
####################################################################################################################################
# backupTestRun
####################################################################################################################################
sub backupTestRun
{
my $strTest = shift;
my $bVmOut = shift;
# If no test was specified, then run them all
if (!defined($strTest))
{
$strTest = 'all';
}
# Setup global variables
my $oHostGroup = hostGroupGet();
my $strTestPath = $oHostGroup->paramGet(HOST_PARAM_TEST_PATH);
my $iProcessMax = $oHostGroup->paramGet(HOST_PARAM_PROCESS_MAX);
# Setup test variables
my $iRun;
my $strModule = 'backup';
my $strThisTest;
my $strStanza = HOST_STANZA;
my $oLogTest;
my $strArchiveChecksum = '72b9da071c13957fb4ca31f05dbd5c644297c2f7';
my $iArchiveMax = 3;
my $strArchiveTestFile = testDataPath() . '/backup.wal2_' . WAL_VERSION_94 . '.bin';
pathModeDefaultSet('0700');
fileModeDefaultSet('0600');
# Print test banner
if (!$bVmOut)
{
&log(INFO, 'BACKUP MODULE ******************************************************************');
&log(INFO, "PROCESS-MAX: ${iProcessMax}\n");
}
#-------------------------------------------------------------------------------------------------------------------------------
# Test archive-push
#-------------------------------------------------------------------------------------------------------------------------------
$strThisTest = 'archive-push';
if ($strTest eq 'all' || $strTest eq $strThisTest)
{
$iRun = 0;
if (!$bVmOut)
{
&log(INFO, "Test ${strThisTest}\n");
}
for (my $bRemote = false; $bRemote <= true; $bRemote++)
{
for (my $bCompress = false; $bCompress <= true; $bCompress++)
{
for (my $bArchiveAsync = false; $bArchiveAsync <= true; $bArchiveAsync++)
{
# Increment the run, log, and decide whether this unit test should be run
if (!testRun(++$iRun,
"rmt ${bRemote}, cmp ${bCompress}, " .
"arc_async ${bArchiveAsync}",
$iProcessMax == 1 ? $strModule : undef,
$iProcessMax == 1 ? $strThisTest: undef,
\$oLogTest)) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = backupTestSetup(
true, $oLogTest, {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => $bArchiveAsync});
# Create the xlog path
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
filePathCreate($strXlogPath, undef, false, true);
my $strCommand =
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() .
' --no-fork --stanza=db archive-push';
# Loop through backups
for (my $iBackup = 1; $iBackup <= 3; $iBackup++)
{
my $strArchiveFile;
# Loop through archive files
for (my $iArchive = 1; $iArchive <= $iArchiveMax; $iArchive++)
{
my $strSourceFile;
# Construct the archive filename
my $iArchiveNo = (($iBackup - 1) * $iArchiveMax + ($iArchive - 1)) + 1;
if ($iArchiveNo > 255)
{
confess 'backup total * archive total cannot be greater than 255';
}
($strArchiveFile, $strSourceFile) =
archiveGenerate($oFile, $strXlogPath, 2, $iArchiveNo, WAL_VERSION_94);
&log(INFO, ' backup ' . sprintf('%02d', $iBackup) .
', archive ' .sprintf('%02x', $iArchive) .
" - ${strArchiveFile}");
my $strArchiveTmp = undef;
if ($iBackup == 1 && $iArchive == 2)
{
# Should succeed when temp file already exists
&log(INFO, ' test archive when tmp file exists');
$strArchiveTmp =
$oHostBackup->repoPath() . "/archive/${strStanza}/" .
PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . "/${strArchiveFile}.pgbackrest.tmp";
executeTest('sudo chmod 770 ' . dirname($strArchiveTmp));
fileStringWrite($strArchiveTmp, 'JUNK');
if ($bRemote)
{
executeTest('sudo chown ' . $oHostBackup->userGet() . " ${strArchiveTmp}");
}
}
$oHostDbMaster->executeSimple(
$strCommand . ($bRemote && $iBackup == $iArchive ? ' --cmd-ssh=/usr/bin/ssh' : '') .
" ${strSourceFile}",
{oLogTest => $oLogTest});
# Make sure the temp file no longer exists
if (defined($strArchiveTmp))
{
my $oWait = waitInit(5);
my $bFound = true;
do
{
$bFound = fileExists($strArchiveTmp);
}
while ($bFound && waitMore($oWait));
if ($bFound)
{
confess "${strArchiveTmp} should have been removed by archive command";
}
}
if ($iArchive == $iBackup)
{
# load the archive info file and munge it for testing by breaking the database version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
&log(INFO, ' test db version mismatch error');
$oHostDbMaster->executeSimple(
$strCommand . " ${strSourceFile}",
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $oLogTest});
# Break the system id
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}});
&log(INFO, ' test db system-id mismatch error');
$oHostDbMaster->executeSimple(
$strCommand . " ${strSourceFile}",
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $oLogTest});
# Restore the file to its original condition
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
# Fail because the process was killed
if ($iBackup == 1 && !$bCompress)
{
&log(INFO, ' test stop');
if ($bArchiveAsync)
{
my $oExecArchive =
$oHostDbMaster->execute(
$strCommand . ' --test --test-delay=5 --test-point=' .
lc(TEST_ARCHIVE_PUSH_ASYNC_START) . "=y ${strSourceFile}",
{oLogTest => $oLogTest, iExpectedExitStatus => ERROR_TERM});
$oExecArchive->begin();
$oExecArchive->end(TEST_ARCHIVE_PUSH_ASYNC_START);
$oHostDbMaster->stop({bForce => true});
$oExecArchive->end();
}
else
{
$oHostDbMaster->stop({strStanza => $oHostDbMaster->stanza()});
}
$oHostDbMaster->executeSimple(
$strCommand . " ${strSourceFile}",
{oLogTest => $oLogTest, iExpectedExitStatus => ERROR_STOP});
$oHostDbMaster->start({strStanza => $bArchiveAsync ? undef : $strStanza});
}
# Should succeed because checksum is the same
&log(INFO, ' test archive duplicate ok');
$oHostDbMaster->executeSimple($strCommand . " ${strSourceFile}", {oLogTest => $oLogTest});
# Now it should break on archive duplication (because checksum is different
&log(INFO, ' test archive duplicate error');
($strArchiveFile, $strSourceFile) =
archiveGenerate($oFile, $strXlogPath, 1, $iArchiveNo, WAL_VERSION_94);
$oHostDbMaster->executeSimple(
$strCommand . " ${strSourceFile}",
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $oLogTest});
if ($bArchiveAsync)
{
my $strDuplicateWal =
($bRemote ? $oHostDbMaster->spoolPath() :
$oHostBackup->repoPath()) .
"/archive/${strStanza}/out/${strArchiveFile}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
fileRemove($strDuplicateWal);
}
# Test .partial archive
&log(INFO, ' test .partial archive');
($strArchiveFile, $strSourceFile) =
archiveGenerate($oFile, $strXlogPath, 2, $iArchiveNo, WAL_VERSION_94, true);
$oHostDbMaster->executeSimple($strCommand . " ${strSourceFile}", {oLogTest => $oLogTest});
archiveCheck($oFile, $strArchiveFile, $strArchiveChecksum, $bCompress);
# Test .partial archive duplicate
&log(INFO, ' test .partial archive duplicate');
$oHostDbMaster->executeSimple($strCommand . " ${strSourceFile}", {oLogTest => $oLogTest});
# Test .partial archive with different checksum
&log(INFO, ' test .partial archive with different checksum');
($strArchiveFile, $strSourceFile) =
archiveGenerate($oFile, $strXlogPath, 1, $iArchiveNo, WAL_VERSION_94, true);
$oHostDbMaster->executeSimple(
$strCommand . " ${strSourceFile}",
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $oLogTest});
if ($bArchiveAsync)
{
my $strDuplicateWal =
($bRemote ? $oHostDbMaster->spoolPath() : $oHostBackup->repoPath()) .
"/archive/${strStanza}/out/${strArchiveFile}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
fileRemove($strDuplicateWal);
}
}
archiveCheck($oFile, $strArchiveFile, $strArchiveChecksum, $bCompress);
}
# Might be nice to add tests for .backup files here (but this is already tested in full backup)
}
if (defined($oLogTest))
{
$oLogTest->supplementalAdd($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
}
}
}
}
testCleanup(\$oLogTest);
}
#-------------------------------------------------------------------------------------------------------------------------------
# Test archive-stop
#-------------------------------------------------------------------------------------------------------------------------------
$strThisTest = 'archive-stop';
if ($strTest eq 'all' || $strTest eq $strThisTest)
{
$iRun = 0;
if (!$bVmOut)
{
&log(INFO, "Test ${strThisTest}\n");
}
for (my $bRemote = false; $bRemote <= true; $bRemote++)
{
for (my $bCompress = false; $bCompress <= true; $bCompress++)
{
for (my $iError = 0; $iError <= $bRemote; $iError++)
{
# Increment the run, log, and decide whether this unit test should be run
if (!testRun(++$iRun,
"rmt ${bRemote}, cmp ${bCompress}" .
', error ' . ($iError ? 'connect' : 'version'),
$iProcessMax == 1 ? $strModule : undef,
$iProcessMax == 1 ? $strThisTest: undef,
\$oLogTest)) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = backupTestSetup(
true, $oLogTest, {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true});
# Create the xlog path
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
filePathCreate($strXlogPath, undef, false, true);
# Push a WAL segment
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 1);
# Break the database version of the archive info file
if ($iError == 0)
{
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
}
# Push two more segments with errors to exceed archive-max-mb
$oHostDbMaster->archivePush(
$strXlogPath, $strArchiveTestFile, 2, $iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
$oHostDbMaster->archivePush(
$strXlogPath, $strArchiveTestFile, 3, $iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
# Now this segment will get dropped
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 4);
# Fix the database version
if ($iError == 0)
{
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
}
# Remove the stop file
fileRemove($oHostDbMaster->spoolPath() . '/stop/db-archive.stop');
# Check the dir to be sure that segment 2 and 3 were not pushed yet
executeTest(
'ls -1R ' . $oHostBackup->repoPath() . "/archive/${strStanza}/" . PG_VERSION_94 . "-1/0000000100000001",
{oLogTest => $oLogTest, bRemote => $bRemote});
# Push segment 5
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 5, undef, false);
# Check that 5 is pushed
executeTest(
'ls -1R ' . $oHostBackup->repoPath() . "/archive/${strStanza}/" . PG_VERSION_94 . "-1/0000000100000001",
{oLogTest => $oLogTest, bRemote => $bRemote});
# Call push without a segment
$oHostDbMaster->archivePush($strXlogPath);
# Check the dir to be sure that segment 2 and 3 were pushed
executeTest(
'ls -1R ' . $oHostBackup->repoPath() . "/archive/${strStanza}/" . PG_VERSION_94 . "-1/0000000100000001",
{oLogTest => $oLogTest, bRemote => $bRemote});
}
}
}
testCleanup(\$oLogTest);
}
#-------------------------------------------------------------------------------------------------------------------------------
# Test archive-get
#-------------------------------------------------------------------------------------------------------------------------------
$strThisTest = 'archive-get';
if ($strTest eq 'all' || $strTest eq $strThisTest)
{
$iRun = 0;
if (!$bVmOut)
{
&log(INFO, "Test ${strThisTest}\n");
}
for (my $bRemote = false; $bRemote <= true; $bRemote++)
{
for (my $bCompress = false; $bCompress <= true; $bCompress++)
{
for (my $bExists = false; $bExists <= true; $bExists++)
{
# Increment the run, log, and decide whether this unit test should be run
if (!testRun(++$iRun,
"rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}",
$iProcessMax == 1 ? $strModule : undef,
$iProcessMax == 1 ? $strThisTest: undef,
\$oLogTest)) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = backupTestSetup(
true, $oLogTest, {bHostBackup => $bRemote, bCompress => $bCompress});
# Create the xlog path
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
filePathCreate($strXlogPath, undef, false, true);
# Create the test path for pg_control and copy pg_control for stanza-create
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
executeTest('cp ' . testDataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
my $strCommand =
$oHostDbMaster->backrestExe() .
' --config=' . $oHostDbMaster->backrestConfig() .
" --stanza=${strStanza} archive-get";
# Fail on missing archive info file
$oHostDbMaster->executeSimple(
$strCommand . " 000000010000000100000001 ${strXlogPath}/000000010000000100000001",
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $oLogTest});
# Create the archive info file
filePathCreate($oFile->pathGet(PATH_BACKUP_ARCHIVE), '0770', undef, true);
(new pgBackRest::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE)))->check(PG_VERSION_94, 6353949018581704918);
if (defined($oLogTest))
{
$oLogTest->supplementalAdd($oFile->pathGet(PATH_BACKUP_ARCHIVE) . '/archive.info');
}
if ($bExists)
{
# Loop through archive files
my $strArchiveFile;
for (my $iArchiveNo = 1; $iArchiveNo <= $iArchiveMax; $iArchiveNo++)
{
# Construct the archive filename
if ($iArchiveNo > 255)
{
confess 'backup total * archive total cannot be greater than 255';
}
$strArchiveFile = uc(sprintf('0000000100000001%08x', $iArchiveNo));
&log(INFO, ' archive ' .sprintf('%02x', $iArchiveNo) .
" - ${strArchiveFile}");
my $strSourceFile = "${strArchiveFile}-${strArchiveChecksum}";
if ($bCompress)
{
$strSourceFile .= '.gz';
}
# Change the directory permissions to enable file creation
executeTest('sudo chmod 770 ' . dirname($oFile->pathGet(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . "-1")));
filePathCreate(
dirname(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . "-1/${strSourceFile}")), '0770', true, true);
$oFile->copy(
PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file $strArchiveTestFile
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . # Destination file
"-1/${strSourceFile}",
false, # Source is not compressed
$bCompress, # Destination compress based on test
undef, undef, # Unused params
'0660', # Mode
true); # Create path if it does not exist
my $strDestinationFile = "${strXlogPath}/${strArchiveFile}";
$oHostDbMaster->executeSimple(
$strCommand . ($bRemote && $iArchiveNo == 1 ? ' --cmd-ssh=/usr/bin/ssh' : '') .
" ${strArchiveFile} ${strDestinationFile}",
{oLogTest => $oLogTest});
# Check that the destination file exists
if ($oFile->exists(PATH_DB_ABSOLUTE, $strDestinationFile))
{
if ($oFile->hash(PATH_DB_ABSOLUTE, $strDestinationFile) ne $strArchiveChecksum)
{
confess "archive file hash does not match ${strArchiveChecksum}";
}
}
else
{
confess 'archive file is not in destination';
}
}
}
else
{
$oHostDbMaster->stop();
$oHostDbMaster->executeSimple(
$strCommand . " 000000090000000900000009 ${strXlogPath}/RECOVERYXLOG",
{iExpectedExitStatus => ERROR_STOP, oLogTest => $oLogTest});
$oHostDbMaster->start();
$oHostDbMaster->executeSimple(
$strCommand . " 000000090000000900000009 ${strXlogPath}/RECOVERYXLOG",
{iExpectedExitStatus => 1, oLogTest => $oLogTest});
}
}
}
}
testCleanup(\$oLogTest);
}
#-------------------------------------------------------------------------------------------------------------------------------
# Test expire
#-------------------------------------------------------------------------------------------------------------------------------
$strThisTest = 'expire';
if ($strTest eq 'all' || $strTest eq $strThisTest)
{
$iRun = 0;
if (!$bVmOut)
{
&log(INFO, "Test ${strThisTest}\n");
}
if (testRun(++$iRun,
"local",
$iProcessMax == 1 ? $strModule : undef,
$iProcessMax == 1 ? $strThisTest: undef,
\$oLogTest))
{
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = backupTestSetup(true, $oLogTest);
# Create the test object
my $oExpireTest = new pgBackRestTest::Backup::Common::ExpireCommonTest($oHostBackup, $oFile, $oLogTest);
# ??? This function creates data elements in the $oExpireTest object that are used by the $oExpireTest functions. But
# should probably change to use the stanza-create command especially with stanza-upgrade.
$oExpireTest->stanzaCreate($strStanza, PG_VERSION_92);
use constant SECONDS_PER_DAY => 86400;
my $lBaseTime = time() - (SECONDS_PER_DAY * 56);
#-----------------------------------------------------------------------------------------------------------------------
my $strDescription = 'Nothing to expire';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY, 246);
$oExpireTest->process($strStanza, 1, 1, BACKUP_TYPE_FULL, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire oldest full backup, archive expire falls on segment major boundary';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->process($strStanza, 1, 1, BACKUP_TYPE_FULL, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire oldest full backup';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY, 256);
$oExpireTest->process($strStanza, 1, 1, BACKUP_TYPE_FULL, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire oldest diff backup, archive expire does not fall on major segment boundary';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY, undef, 0);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY, undef, 0);
$oExpireTest->process($strStanza, 1, 1, BACKUP_TYPE_DIFF, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire oldest diff backup (cascade to incr)';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->process($strStanza, 1, 1, BACKUP_TYPE_DIFF, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire archive based on newest incr backup';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->process($strStanza, 1, 1, BACKUP_TYPE_INCR, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire diff treating full as diff';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->process($strStanza, 2, 1, BACKUP_TYPE_DIFF, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire diff with retention-archive with warning retention-diff not set';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->process($strStanza, undef, undef, BACKUP_TYPE_DIFF, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire full with retention-archive with warning retention-full not set';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->process($strStanza, undef, undef, BACKUP_TYPE_FULL, 1, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire no archive with warning since retention-archive not set for INCR';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->process($strStanza, 1, 1, BACKUP_TYPE_INCR, undef, $strDescription);
#-----------------------------------------------------------------------------------------------------------------------
$strDescription = 'Expire no archive with warning since neither retention-archive nor retention-diff is set';
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->backupCreate($strStanza, BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY);
$oExpireTest->process($strStanza, undef, undef, BACKUP_TYPE_DIFF, undef, $strDescription);
testCleanup(\$oLogTest);
}
}
#-------------------------------------------------------------------------------------------------------------------------------
# Test synthetic
#
# Check the backup and restore functionality using synthetic data.
#-------------------------------------------------------------------------------------------------------------------------------
$strThisTest = 'synthetic';
if ($strTest eq 'all' || $strTest eq $strThisTest)
{
$iRun = 0;
if (!$bVmOut)
{
&log(INFO, "Test ${strModule} - ${strThisTest}\n");
}
for (my $bRemote = false; $bRemote <= true; $bRemote++)
{
for (my $bCompress = false; $bCompress <= true; $bCompress++)
{
for (my $bHardLink = false; $bHardLink <= true; $bHardLink++)
{
# Increment the run, log, and decide whether this unit test should be run
if (!testRun(++$iRun,
"rmt ${bRemote}, cmp ${bCompress}, hardlink ${bHardLink}",
$iProcessMax == 1 ? $strModule : undef,
$iProcessMax == 1 ? $strThisTest: undef,
\$oLogTest)) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = backupTestSetup(
true, $oLogTest, {bHostBackup => $bRemote, bCompress => $bCompress, bHardLink => $bHardLink});
# Determine if this is a neutral test, i.e. we only want to do it once for local and once for remote. Neutral means
# that options such as compression and hardlinks are disabled
my $bNeutralTest = !$bCompress && !$bHardLink;
# Get base time
my $lTime = time() - 10000;
# Build the manifest
my %oManifest;
$oManifest{&INI_SECTION_BACKREST}{&INI_KEY_VERSION} = BACKREST_VERSION;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY} = JSON::PP::false;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = $bCompress ? JSON::PP::true : JSON::PP::false;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = $bHardLink ? JSON::PP::true : JSON::PP::false;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} = JSON::PP::false;
$oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG} = 201409291;
$oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CONTROL} = 942;
$oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_SYSTEM_ID} = 6353949018581704918;
$oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} = PG_VERSION_94;
$oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_ID} = 1;
$oManifest{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} =
$oHostDbMaster->dbBasePath();
$oManifest{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_PATH;
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_PGVERSION, PG_VERSION_94,
'184473f470864e067ee3a22e64b47b0a1c356f29', $lTime, undef, true);
# Create base path
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base');
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/1');
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/1/12000', 'BASE',
'a3b357a3e395e43fcfb19bb13f3c1b5179279593', $lTime);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/1/' . DB_FILE_PGVERSION,
PG_VERSION_94, '184473f470864e067ee3a22e64b47b0a1c356f29', $lTime, '660');
if ($bNeutralTest && !$bRemote)
{
executeTest('sudo chown 7777 ' . $oHostDbMaster->dbBasePath() . '/base/1/' . DB_FILE_PGVERSION);
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/1/' . DB_FILE_PGVERSION}
{&MANIFEST_SUBKEY_USER} = INI_FALSE;
}
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384');
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000', 'BASE',
'a3b357a3e395e43fcfb19bb13f3c1b5179279593', $lTime);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/' . DB_FILE_PGVERSION,
PG_VERSION_94, '184473f470864e067ee3a22e64b47b0a1c356f29', $lTime);
if ($bNeutralTest && !$bRemote)
{
executeTest('sudo chown :7777 ' . $oHostDbMaster->dbBasePath() . '/base/16384/' . DB_FILE_PGVERSION);
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/16384/' . DB_FILE_PGVERSION}
{&MANIFEST_SUBKEY_GROUP} = INI_FALSE;
}
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768');
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/33000', '33000',
'7f4c74dc10f61eef43e6ae642606627df1999b34', $lTime);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/' . DB_FILE_PGVERSION,
PG_VERSION_94, '184473f470864e067ee3a22e64b47b0a1c356f29', $lTime);
# Create global path
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'global');
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_PGCONTROL, '[replaceme]',
'89373d9f2973502940de06bc5212489df3f8a912', $lTime - 100, undef, true);
# Copy pg_control
executeTest(
'cp ' . testDataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
utime($lTime - 100, $lTime - 100, $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL)
or confess &log(ERROR, "unable to set time");
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL}
{&MANIFEST_SUBKEY_SIZE} = 8192;
# Create tablespace path
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGTBLSPC);
# Create paths/files to ignore
if ($bNeutralTest && !$bRemote)
{
# Create temp dir and file that will be ignored
$oHostDbMaster->dbPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/' . DB_FILE_PREFIX_TMP);
$oHostDbMaster->dbFileCreate(
\%oManifest, MANIFEST_TARGET_PGDATA, 'base/' . DB_FILE_PREFIX_TMP . '/' . DB_FILE_PREFIX_TMP . '.1', 'IGNORE');
# Create pg_dynshmem dir and file - only file will be ignored
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGDYNSHMEM);
# $oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGDYNSHMEM . '/anything.tmp', 'IGNORE');
# Create pg_notify dir and file - only file will be ignored
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGNOTIFY);
# $oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGNOTIFY . '/anything.tmp', 'IGNORE');
# Create pg_replslot dir and file - only file will be ignored
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGREPLSLOT);
$oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGREPLSLOT . '/anything.tmp', 'IGNORE');
# Create pg_serial dir and file - only file will be ignored
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSERIAL);
# $oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSERIAL . '/anything.tmp', 'IGNORE');
# Create pg_snaphots dir and file - only file will be ignored
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSNAPSHOTS);
# $oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSNAPSHOTS . '/anything.tmp', 'IGNORE');
# Create pg_stat_tmp dir and file - only file will be ignored
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSTATTMP);
$oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSTATTMP . '/anything.tmp', 'IGNORE');
# Create pg_subtrans dir and file - only file will be ignored
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSUBTRANS);
$oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSUBTRANS . '/anything.tmp', 'IGNORE');
# More files to ignore
$oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_POSTGRESQLAUTOCONFTMP, 'IGNORE');
$oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_POSTMASTEROPTS, 'IGNORE');
$oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_RECOVERYCONF, 'IGNORE');
$oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_RECOVERYDONE, 'IGNORE');
}
# Backup Info (with no stanzas)
#-----------------------------------------------------------------------------------------------------------------------
$oHostDbMaster->info('no stanzas exist');
$oHostDbMaster->info('no stanzas exist', {strOutput => INFO_OUTPUT_JSON});
# Full backup
#-----------------------------------------------------------------------------------------------------------------------
my $strType = BACKUP_TYPE_FULL;
my $strOptionalParam = '--manifest-save-threshold=3';
my $strTestPoint;
if ($bNeutralTest && $bRemote)
{
$strOptionalParam .= ' --protocol-timeout=2 --db-timeout=1';
if ($iProcessMax > 1)
{
$strTestPoint = TEST_KEEP_ALIVE;
}
}
# Create a file link
filePathCreate($oHostDbMaster->dbPath() . '/pg_config', undef, undef, true);
testFileCreate(
$oHostDbMaster->dbPath() . '/pg_config/postgresql.conf', "listen_addresses = *\n", $lTime - 100);
testLinkCreate($oHostDbMaster->dbPath() . '/pg_config/postgresql.conf.link', './postgresql.conf');
$oHostDbMaster->manifestLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.conf',
'../pg_config/postgresql.conf', true);
# This link will cause errors because it points to the same location as above
$oHostDbMaster->manifestLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_config_bad',
'../../db/pg_config');
my $strFullBackup = $oHostBackup->backup(
$strType, 'error on identical link destinations',
{oExpectedManifest => \%oManifest, strOptionalParam => '--log-level-console=detail',
iExpectedExitStatus => ERROR_LINK_DESTINATION});
# Remove failing link
$oHostDbMaster->manifestLinkRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_config_bad');
# This link will fail because it points to a link
$oHostDbMaster->manifestLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.conf.bad',
'../pg_config/postgresql.conf.link');
# Fail bacause two links point to the same place
$strFullBackup = $oHostBackup->backup(
$strType, 'error on link to a link',
{oExpectedManifest => \%oManifest, strOptionalParam => '--log-level-console=detail',
iExpectedExitStatus => ERROR_LINK_DESTINATION});
# Remove failing link
$oHostDbMaster->manifestLinkRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.conf.bad');
# Create stat directory link and file
filePathCreate($oHostDbMaster->dbPath() . '/pg_stat', undef, undef, true);
$oHostDbMaster->manifestLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_stat', '../pg_stat');
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA . '/pg_stat', 'global.stat', 'stats',
'e350d5ce0153f3e22d5db21cf2a4eff00f3ee877', $lTime - 100, undef, true);
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_clog');
$strFullBackup = $oHostBackup->backup(
$strType, 'create pg_stat link, pg_clog dir',
{oExpectedManifest => \%oManifest,
strOptionalParam => $strOptionalParam . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : ''),
strTest => $strTestPoint,
fTestDelay => 0});
# Error on backup option to check logging
#-----------------------------------------------------------------------------------------------------------------------
if ($bNeutralTest && !$bRemote)
{
$oHostBackup->backup(
$strType, 'invalid cmd line',
{oExpectedManifest => \%oManifest, strStanza => 'bogus', iExpectedExitStatus => ERROR_OPTION_REQUIRED});
}
# Test protocol timeout
#-----------------------------------------------------------------------------------------------------------------------
if ($bNeutralTest && $bRemote)
{
$oHostBackup->backup(
$strType, 'protocol timeout',
{oExpectedManifest => \%oManifest, strOptionalParam => '--protocol-timeout=1 --db-timeout=.1',
strTest => TEST_BACKUP_START, fTestDelay => 1, iExpectedExitStatus => ERROR_PROTOCOL_TIMEOUT});
# Remove the aborted backup so the next backup is not a resume
testPathRemove($oHostBackup->repoPath() . "/temp/${strStanza}.tmp");
}
# Stop operations and make sure the correct error occurs
#-----------------------------------------------------------------------------------------------------------------------
if ($bNeutralTest)
{
# Test a backup abort
my $oExecuteBackup = $oHostBackup->backupBegin(
$strType, 'abort backup - local',
{oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_START, fTestDelay => 5,
iExpectedExitStatus => ERROR_TERM});
$oHostDbMaster->stop({bForce => true});
$oHostBackup->backupEnd($strType, $oExecuteBackup, {oExpectedManifest => \%oManifest});
# Test global stop
$oHostBackup->backup(
$strType, 'global stop',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_STOP});
# Test stanza stop
$oHostDbMaster->stop({strStanza => $oHostDbMaster->stanza()});
# This time a warning should be generated
$oHostDbMaster->stop({strStanza => $oHostDbMaster->stanza()});
$oHostBackup->backup(
$strType, 'stanza stop',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_STOP});
$oHostDbMaster->start({strStanza => $strStanza});
$oHostDbMaster->start();
# This time a warning should be generated
$oHostDbMaster->start();
# If the backup is remote then test remote stops
if ($bRemote)
{
my $oExecuteBackup = $oHostBackup->backupBegin(
$strType, 'abort backup - remote',
{oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_START, fTestDelay => 5,
iExpectedExitStatus => ERROR_TERM});
$oHostBackup->stop({bForce => true});
$oHostBackup->backupEnd($strType, $oExecuteBackup, {oExpectedManifest => \%oManifest});
$oHostBackup->backup(
$strType, 'global stop',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_STOP});
$oHostBackup->start();
}
}
# Cleanup any garbage left in the temp backup path
executeTest(
'sudo rm -rf ' . $oHostBackup->repoPath() . "/temp/${strStanza}.tmp", {bRemote => $bRemote});
# Resume Full Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_FULL;
# These files should never be backed up (this requires the next backup to do --force)
testFileCreate($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID, 'JUNK');
testFileCreate($oHostDbMaster->dbBasePath() . '/' . DB_FILE_BACKUPLABELOLD, 'JUNK');
testFileCreate($oHostDbMaster->dbBasePath() . '/' . DB_FILE_RECOVERYCONF, 'JUNK');
testFileCreate($oHostDbMaster->dbBasePath() . '/' . DB_FILE_RECOVERYDONE, 'JUNK');
# Create files in root tblspc paths that should not be copied or deleted.
# This will be checked later after a --force restore.
my $strDoNotDeleteFile = $oHostDbMaster->tablespacePath(1, 2) . '/donotdelete.txt';
filePathCreate(dirname($strDoNotDeleteFile), undef, undef, true);
testFileCreate($strDoNotDeleteFile, 'DONOTDELETE-1-2');
filePathCreate($oHostDbMaster->tablespacePath(1), undef, undef, true);
testFileCreate($oHostDbMaster->tablespacePath(1) . '/donotdelete.txt', 'DONOTDELETE-1');
filePathCreate($oHostDbMaster->tablespacePath(2), undef, undef, true);
testFileCreate($oHostDbMaster->tablespacePath(2) . '/donotdelete.txt', 'DONOTDELETE-2');
filePathCreate($oHostDbMaster->tablespacePath(2, 2), undef, undef, true);
testFileCreate($oHostDbMaster->tablespacePath(2, 2) . '/donotdelete.txt', 'DONOTDELETE-2-2');
filePathCreate($oHostDbMaster->tablespacePath(11), undef, undef, true);
my $strTmpPath = $oHostBackup->repoPath() . "/temp/${strStanza}.tmp";
executeTest("sudo chmod g+w " . dirname($strTmpPath));
testPathMove($oHostBackup->repoPath() . "/backup/${strStanza}/${strFullBackup}", $strTmpPath);
my $oMungeManifest = new pgBackRest::Manifest("$strTmpPath/backup.manifest");
$oMungeManifest->remove(MANIFEST_SECTION_TARGET_FILE, MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGVERSION, 'checksum');
$oMungeManifest->save();
# Create a temp file in backup temp root to be sure it's deleted correctly
executeTest("touch ${strTmpPath}/file.tmp" . ($bCompress ? '.gz' : ''),
{bRemote => $bRemote});
executeTest("sudo chmod -R g+w " . dirname($strTmpPath));
$strFullBackup = $oHostBackup->backup(
$strType, 'resume',
{oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_RESUME, strOptionalParam => '--force'});
# Remove postmaster.pid so restore will succeed (the rest will be cleaned up)
testFileRemove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
# Misconfigure repo-path and check errors
#-----------------------------------------------------------------------------------------------------------------------
if ($bNeutralTest)
{
$oHostBackup->backup(
$strType, 'invalid repo',
{oExpectedManifest => \%oManifest, strOptionalParam => '--' . OPTION_REPO_PATH . '=/bogus_path' .
' --log-level-console=detail', iExpectedExitStatus => ERROR_PATH_MISSING});
}
# Restore - tests various mode, extra files/paths, missing files/paths
#-----------------------------------------------------------------------------------------------------------------------
my $bDelta = true;
my $bForce = false;
# Munge permissions/modes on files that will be fixed by the restore
if ($bNeutralTest && !$bRemote)
{
executeTest("sudo chown :7777 " . $oHostDbMaster->dbBasePath() . '/base/1/' . DB_FILE_PGVERSION);
executeTest("sudo chmod 600 " . $oHostDbMaster->dbBasePath() . '/base/1/' . DB_FILE_PGVERSION);
}
# Create a path and file that are not in the manifest
$oHostDbMaster->dbPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'deleteme');
$oHostDbMaster->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'deleteme/deleteme.txt', 'DELETEME');
# Change path mode
$oHostDbMaster->dbPathMode(\%oManifest, MANIFEST_TARGET_PGDATA, 'base', '0777');
# Remove a path
$oHostDbMaster->dbPathRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_clog');
# Remove a file
$oHostDbMaster->dbFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000');
# Restore will reset invalid user and group so do the same in the manifest
if ($bNeutralTest && !$bRemote)
{
delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/1/' . DB_FILE_PGVERSION}
{&MANIFEST_SUBKEY_USER});
delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/16384/' . DB_FILE_PGVERSION}
{&MANIFEST_SUBKEY_GROUP});
}
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'add and delete files', undef, ' --link-all' . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : ''),
undef, $bNeutralTest && !$bRemote ? 'root' : undef);
# Fix permissions on the restore log & remove lock files
if ($bNeutralTest && !$bRemote)
{
executeTest('sudo chown -R ' . TEST_USER . ':' . POSTGRES_GROUP . ' ' . $oHostBackup->logPath());
executeTest('sudo rm -rf ' . $oHostDbMaster->lockPath() . '/*');
}
# Change an existing link to the wrong directory
$oHostDbMaster->dbFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_stat');
$oHostDbMaster->dbLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_stat', '../wrong');
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'fix broken symlink', undef, ' --link-all --log-level-console=detail');
# Additional restore tests that don't need to be performed for every permutation
if ($bNeutralTest && !$bRemote)
{
# This time manually restore all links
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'restore all links by mapping', undef, '--log-level-console=detail' .
' --link-map=pg_stat=../pg_stat --link-map=postgresql.conf=../pg_config/postgresql.conf');
# Error when links overlap
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'restore all links by mapping', ERROR_LINK_DESTINATION, '--log-level-console=warn' .
' --link-map=pg_stat=../pg_stat --link-map=postgresql.conf=../pg_stat/postgresql.conf');
# Error when links still exist on non-delta restore
$bDelta = false;
executeTest('rm -rf ' . $oHostDbMaster->dbBasePath() . "/*");
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'error on existing linked path', ERROR_RESTORE_PATH_NOT_EMPTY, '--log-level-console=warn --link-all');
executeTest('rm -rf ' . $oHostDbMaster->dbPath() . "/pg_stat/*");
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'error on existing linked file', ERROR_RESTORE_PATH_NOT_EMPTY, '--log-level-console=warn --link-all');
# Now a combination of remapping
$bDelta = true;
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'restore all links --link-all and mapping', undef,
'--log-level-console=detail --link-map=pg_stat=../pg_stat --link-all');
}
# Restore - test errors when $PGDATA cannot be verified
#-----------------------------------------------------------------------------------------------------------------------
$bDelta = true;
$bForce = true;
# Remove PG_VERSION
$oHostDbMaster->dbFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_PGVERSION);
# Attempt the restore
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'fail on missing ' . DB_FILE_PGVERSION, ERROR_RESTORE_PATH_NOT_EMPTY, '--log-level-console=detail');
# Write a backup.manifest file to make $PGDATA valid
testFileCreate($oHostDbMaster->dbBasePath() . '/backup.manifest', 'BOGUS');
# Munge the user to make sure it gets reset on the next run
$oHostBackup->manifestMunge(
$strFullBackup, MANIFEST_SECTION_TARGET_FILE, MANIFEST_FILE_PGCONTROL, MANIFEST_SUBKEY_USER, 'bogus');
$oHostBackup->manifestMunge(
$strFullBackup, MANIFEST_SECTION_TARGET_FILE, MANIFEST_FILE_PGCONTROL, MANIFEST_SUBKEY_GROUP, 'bogus');
# Restore succeeds
$oHostDbMaster->manifestLinkMap(\%oManifest, MANIFEST_TARGET_PGDATA . '/pg_stat');
$oHostDbMaster->manifestLinkMap(\%oManifest, MANIFEST_TARGET_PGDATA . '/postgresql.conf');
$oHostDbMaster->restore(
$strFullBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'restore succeeds with backup.manifest file', undef, '--log-level-console=detail');
# Various broken info tests
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
$oHostDbMaster->manifestReference(\%oManifest, $strFullBackup);
# Break the database version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_DB_VERSION => '8.0'}});
$oHostBackup->backup(
$strType, 'invalid database version',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_BACKUP_MISMATCH,
strOptionalParam => '--log-level-console=detail'});
# Break the database system id
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 6999999999999999999}});
$oHostBackup->backup(
$strType, 'invalid system id',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_BACKUP_MISMATCH,
strOptionalParam => '--log-level-console=detail'});
# Break the control version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_CONTROL => 842}});
$oHostBackup->backup(
$strType, 'invalid control version',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_BACKUP_MISMATCH,
strOptionalParam => '--log-level-console=detail'});
# Break the catalog version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_CATALOG => 197208141}});
$oHostBackup->backup(
$strType, 'invalid catalog version',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_BACKUP_MISMATCH,
strOptionalParam => '--log-level-console=detail'});
# Restore the file to its original condition
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
# Test broken tablespace configuration
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
my $strTblSpcPath = $oHostDbMaster->dbBasePath() . '/' . DB_PATH_PGTBLSPC;
# Create a directory in pg_tablespace
filePathCreate("${strTblSpcPath}/path");
$oHostBackup->backup(
$strType, 'invalid path in ' . DB_PATH_PGTBLSPC,
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_LINK_EXPECTED,
strOptionalParam => '--log-level-console=detail'});
testPathRemove("${strTblSpcPath}/path");
if ($bNeutralTest && !$bRemote)
{
# Create a relative link in PGDATA
testLinkCreate("${strTblSpcPath}/99999", '../');
$oHostBackup->backup(
$strType, 'invalid relative tablespace is ../',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_TABLESPACE_IN_PGDATA,
strOptionalParam => '--log-level-console=detail'});
testFileRemove("${strTblSpcPath}/99999");
testLinkCreate("${strTblSpcPath}/99999", '..');
$oHostBackup->backup(
$strType, 'invalid relative tablespace is ..',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_TABLESPACE_IN_PGDATA,
strOptionalParam => '--log-level-console=detail'});
testFileRemove("${strTblSpcPath}/99999");
testLinkCreate("${strTblSpcPath}/99999", '../../base/');
$oHostBackup->backup(
$strType, 'invalid relative tablespace is ../../$PGDATA',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_TABLESPACE_IN_PGDATA,
strOptionalParam => '--log-level-console=detail'});
testFileRemove("${strTblSpcPath}/99999");
testLinkCreate("${strTblSpcPath}/99999", '../../base');
$oHostBackup->backup(
$strType, 'invalid relative tablespace is ../../$PGDATA',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_TABLESPACE_IN_PGDATA,
strOptionalParam => '--log-level-console=detail'});
testFileRemove("${strTblSpcPath}/99999");
# Create a link to a link
testLinkCreate($oHostDbMaster->dbPath() . "/intermediate_link", $oHostDbMaster->dbPath() . '/tablespace/ts1');
testLinkCreate("${strTblSpcPath}/99999", $oHostDbMaster->dbPath() . "/intermediate_link");
$oHostBackup->backup(
$strType, 'tablespace link references a link',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_LINK_DESTINATION,
strOptionalParam => '--log-level-console=detail'});
testFileRemove($oHostDbMaster->dbPath() . "/intermediate_link");
testFileRemove("${strTblSpcPath}/99999");
}
# Create a relative link in PGDATA
testLinkCreate("${strTblSpcPath}/99999", '../invalid_tblspc');
$oHostBackup->backup(
$strType, 'invalid relative tablespace in $PGDATA',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_TABLESPACE_IN_PGDATA,
strOptionalParam => '--log-level-console=detail'});
testFileRemove("${strTblSpcPath}/99999");
# Create tablespace with same initial dir name as $PGDATA
if ($bNeutralTest && !$bRemote)
{
testLinkCreate("${strTblSpcPath}/99999", $oHostDbMaster->dbBasePath() . '_tbs');
$oHostBackup->backup(
$strType, '$PGDATA is a substring of valid tblspc excluding / (file open err expected)',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_FILE_OPEN,
strOptionalParam => '--log-level-console=detail'});
testFileRemove("${strTblSpcPath}/99999");
}
# Create tablespace in PGDATA
testLinkCreate("${strTblSpcPath}/99999", $oHostDbMaster->dbBasePath() . '/invalid_tblspc');
$oHostBackup->backup(
$strType, 'invalid tablespace in $PGDATA',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_TABLESPACE_IN_PGDATA,
strOptionalParam => '--log-level-console=detail'});
testFileRemove("${strTblSpcPath}/99999");
# Incr backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
# Add tablespace 1
$oHostDbMaster->manifestTablespaceCreate(\%oManifest, 1);
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', '16384');
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', '16384/tablespace1.txt', 'TBLSPC1',
'd85de07d6421d90aa9191c11c889bfde43680f0f', $lTime);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'badchecksum.txt', 'BADCHECKSUM',
'f927212cd08d11a42a666b2f04235398e9ceeb51', $lTime, undef, true);
# Create temp dir and file that will be ignored
if ($bNeutralTest && !$bRemote)
{
$oHostDbMaster->dbPathCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', DB_FILE_PREFIX_TMP);
$oHostDbMaster->dbFileCreate(
\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', DB_FILE_PREFIX_TMP . '/' . DB_FILE_PREFIX_TMP . '.1', 'IGNORE');
}
my $strBackup = $oHostBackup->backup(
$strType, 'add tablespace 1', {oExpectedManifest => \%oManifest, strOptionalParam => '--test'});
# Resume Incr Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
# Move database from backup to temp
$strTmpPath = $oHostBackup->repoPath() . "/temp/${strStanza}.tmp";
testPathMove($oHostBackup->repoPath() . "/backup/${strStanza}/${strBackup}", $strTmpPath);
executeTest("sudo chmod -R g+w " . dirname($strTmpPath));
$oMungeManifest = new pgBackRest::Manifest("$strTmpPath/" . FILE_MANIFEST);
$oMungeManifest->set(MANIFEST_SECTION_TARGET_FILE, MANIFEST_TARGET_PGDATA . '/badchecksum.txt', 'checksum', 'bogus');
$oMungeManifest->save();
# Add tablespace 2
$oHostDbMaster->manifestTablespaceCreate(\%oManifest, 2);
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768');
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2.txt', 'TBLSPC2',
'dc7f76e43c46101b47acc55ae4d593a9e6983578', $lTime);
# Also create tablespace 11 to be sure it does not conflict with path of tablespace 1
if ($bNeutralTest)
{
$oHostDbMaster->manifestTablespaceCreate(\%oManifest, 11);
}
$strBackup = $oHostBackup->backup(
$strType, 'resume and add tablespace 2', {oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_RESUME});
# Resume Diff Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_DIFF;
# Drop tablespace 11
if ($bNeutralTest)
{
$oHostDbMaster->manifestTablespaceDrop(\%oManifest, 11);
}
$strTmpPath = $oHostBackup->repoPath() . "/temp/${strStanza}.tmp";
testPathMove($oHostBackup->repoPath() . "/backup/${strStanza}/${strBackup}", $strTmpPath);
executeTest("sudo chmod -R g+w " . dirname($strTmpPath));
$strBackup = $oHostBackup->backup(
$strType, 'cannot resume - new diff',
{oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_NORESUME,
strOptionalParam => '--log-level-console=detail'});
# Resume Diff Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_DIFF;
$strTmpPath = $oHostBackup->repoPath() . "/temp/${strStanza}.tmp";
testPathMove($oHostBackup->repoPath() . "/backup/${strStanza}/${strBackup}", $strTmpPath);
executeTest("sudo chmod -R g+w " . dirname($strTmpPath));
$strBackup = $oHostBackup->backup(
$strType, 'cannot resume - disabled / no repo link',
{oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_NORESUME, bRepoLink => false,
strOptionalParam => '--no-resume --log-level-console=detail'});
# Restore
#-----------------------------------------------------------------------------------------------------------------------
$bDelta = false;
$bForce = false;
# Fail on used path
$oHostDbMaster->restore(
$strBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'fail on used path', ERROR_RESTORE_PATH_NOT_EMPTY, '--log-level-console=detail');
# Fail on undef format
$oHostBackup->manifestMunge($strBackup, INI_SECTION_BACKREST, INI_KEY_FORMAT);
$oHostDbMaster->restore(
$strBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'fail on undef format', ERROR_FORMAT, '--log-level-console=detail');
# Fail on mismatch format
$oHostBackup->manifestMunge($strBackup, INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, 0);
$oHostDbMaster->restore(
$strBackup, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'fail on mismatch format', ERROR_FORMAT, '--log-level-console=detail');
$oHostBackup->manifestMunge($strBackup, INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, BACKREST_FORMAT);
# Remap the base and tablespace paths
my %oRemapHash;
$oRemapHash{&MANIFEST_TARGET_PGDATA} = $oHostDbMaster->dbBasePath(2);
filePathCreate($oHostDbMaster->dbBasePath(2));
$oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/1'} = $oHostDbMaster->tablespacePath(1, 2);
$oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'} = $oHostDbMaster->tablespacePath(2, 2);
$oHostDbMaster->restore(
$strBackup, \%oManifest, \%oRemapHash, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'remap all paths', undef, '--log-level-console=detail');
# Restore (make sure file in root tablespace path is not deleted by --delta)
#-----------------------------------------------------------------------------------------------------------------------
$bDelta = true;
$oHostDbMaster->restore(
$strBackup, \%oManifest, \%oRemapHash, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'ensure file in tblspc root remains after --delta', undef, '--log-level-console=detail');
if (!-e $strDoNotDeleteFile)
{
confess "${strDoNotDeleteFile} was deleted by --delta";
}
# Incr Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
$oHostDbMaster->manifestReference(\%oManifest, $strBackup);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/base2.txt', 'BASE2',
'09b5e31766be1dba1ec27de82f975c1b6eea2a92', $lTime);
$oHostDbMaster->manifestTablespaceDrop(\%oManifest, 1, 2);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2b.txt',
'TBLSPC2B', 'e324463005236d83e6e54795dbddd20a74533bf3', $lTime);
# Munge the version to make sure it gets corrected on the next run
$oHostBackup->manifestMunge($strBackup, INI_SECTION_BACKREST, INI_KEY_VERSION, undef, '0.00');
$strBackup = $oHostBackup->backup(
$strType, 'add files and remove tablespace 2',
{oExpectedManifest => \%oManifest, strOptionalParam => '--log-level-console=detail'});
# Incr Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
$oHostDbMaster->manifestReference(\%oManifest, $strBackup);
# Delete the backup.info and make sure it gets reconstructed correctly
if ($bNeutralTest)
{
executeTest('sudo rm ' . $oHostBackup->repoPath() . "/backup/${strStanza}/backup.info");
}
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000', 'BASEUPDT',
'9a53d532e27785e681766c98516a5e93f096a501', $lTime);
$strBackup =$oHostBackup->backup(
$strType, 'update files',
{oExpectedManifest => \%oManifest, strOptionalParam => '--log-level-console=detail'});
# Diff Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_DIFF;
$oHostDbMaster->manifestReference(\%oManifest, $strFullBackup, true);
$strBackup = $oHostBackup->backup(
$strType, 'updates since last full',
{oExpectedManifest => \%oManifest, strOptionalParam => '--log-level-console=detail'});
# Incr Backup
#
# Remove a file from the db after the manifest has been built but before files are copied. The file will not be shown
# as removed in the log because it had not changed since the last backup so it will only be referenced. This test also
# checks that everything works when there are no jobs to run.
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
$oHostDbMaster->manifestReference(\%oManifest, $strBackup);
my $oBackupExecute = $oHostBackup->backupBegin(
$strType, 'remove files - but won\'t affect manifest',
{oExpectedManifest => \%oManifest, strTest => TEST_MANIFEST_BUILD, fTestDelay => 1,
strOptionalParam => '--log-level-console=detail'});
$oHostDbMaster->dbFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000');
$strBackup = $oHostBackup->backupEnd($strType, $oBackupExecute, {oExpectedManifest => \%oManifest});
# Diff Backup
#
# Remove base2.txt and changed tablespace2c.txt during the backup. The removed file should be logged and the changed
# file should have the new, larger size logged and in the manifest.
#-----------------------------------------------------------------------------------------------------------------------
$oHostDbMaster->manifestReference(\%oManifest, $strFullBackup, true);
$strType = BACKUP_TYPE_DIFF;
$oHostDbMaster->manifestFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000');
$oHostDbMaster->manifestFileRemove(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2b.txt', true);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2c.txt', 'TBLSPC2C',
'ad7df329ab97a1e7d35f1ff0351c079319121836', $lTime);
$oBackupExecute = $oHostBackup->backupBegin(
$strType, 'remove files during backup',
{oExpectedManifest => \%oManifest, strTest => TEST_MANIFEST_BUILD, fTestDelay => 1,
strOptionalParam => '--log-level-console=detail'});
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2c.txt',
'TBLSPCBIGGER', 'dfcb8679956b734706cf87259d50c88f83e80e66', $lTime);
$oHostDbMaster->manifestFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/base2.txt', true);
$strBackup = $oHostBackup->backupEnd($strType, $oBackupExecute, {oExpectedManifest => \%oManifest});
# Full Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_FULL;
$oHostDbMaster->manifestReference(\%oManifest);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000', 'BASEUPDT2',
'7579ada0808d7f98087a0a586d0df9de009cdc33', $lTime);
$strFullBackup = $oHostBackup->backup(
$strType, 'update file',
{oExpectedManifest => \%oManifest, strOptionalParam => '--log-level-console=detail'});
# Backup Info
#-----------------------------------------------------------------------------------------------------------------------
$oHostDbMaster->info('normal output', {strStanza => $oHostDbMaster->stanza()});
$oHostBackup->info('normal output', {strStanza => $oHostBackup->stanza(), strOutput => INFO_OUTPUT_JSON});
# Call expire
#-----------------------------------------------------------------------------------------------------------------------
if ($bRemote && !$bCompress)
{
$oHostBackup->expire({iRetentionFull => 1});
}
else
{
$oHostDbMaster->expire(
{iRetentionFull => 1, iExpectedExitStatus => $bRemote && $bCompress ? ERROR_HOST_INVALID : undef});
}
# Diff Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_DIFF;
$oHostDbMaster->manifestReference(\%oManifest, $strFullBackup);
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/base2.txt', 'BASE2UPDT',
'cafac3c59553f2cfde41ce2e62e7662295f108c0', $lTime);
$strBackup = $oHostBackup->backup(
$strType, 'add file', {oExpectedManifest => \%oManifest, strOptionalParam => '--log-level-console=detail'});
# Selective Restore
#-----------------------------------------------------------------------------------------------------------------------
$bDelta = true;
# Remove mapping for tablespace 1
delete($oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/1'});
# Remove checksum to match zeroed files
delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/32768/33000'}{&MANIFEST_SUBKEY_CHECKSUM});
delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_tblspc/2/PG_9.4_201409291/32768/tablespace2.txt'}
{&MANIFEST_SUBKEY_CHECKSUM});
delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_tblspc/2/PG_9.4_201409291/32768/tablespace2c.txt'}
{&MANIFEST_SUBKEY_CHECKSUM});
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, \%oManifest, \%oRemapHash, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'selective restore 16384', undef, '--log-level-console=detail --db-include=16384');
# Restore checksum values for next test
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/32768/33000'}{&MANIFEST_SUBKEY_CHECKSUM} =
'7f4c74dc10f61eef43e6ae642606627df1999b34';
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_tblspc/2/PG_9.4_201409291/32768/tablespace2.txt'}
{&MANIFEST_SUBKEY_CHECKSUM} = 'dc7f76e43c46101b47acc55ae4d593a9e6983578';
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_tblspc/2/PG_9.4_201409291/32768/tablespace2c.txt'}
{&MANIFEST_SUBKEY_CHECKSUM} = 'dfcb8679956b734706cf87259d50c88f83e80e66';
# Remove checksum to match zeroed file
delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/16384/17000'}{&MANIFEST_SUBKEY_CHECKSUM});
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, \%oManifest, \%oRemapHash, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'selective restore 32768', undef, '--log-level-console=detail --db-include=32768');
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/16384/17000'}{&MANIFEST_SUBKEY_CHECKSUM} =
'7579ada0808d7f98087a0a586d0df9de009cdc33';
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, \%oManifest, \%oRemapHash,
$bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'error on invalid id', ERROR_DB_MISSING, '--log-level-console=warn --db-include=7777');
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, \%oManifest, \%oRemapHash, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'error on system id', ERROR_DB_INVALID, '--log-level-console=warn --db-include=1');
# Compact Restore
#-----------------------------------------------------------------------------------------------------------------------
$bDelta = false;
executeTest('rm -rf ' . $oHostDbMaster->dbBasePath(2) . "/*");
my $strDbPath = $oHostDbMaster->dbBasePath(2) . '/base';
filePathCreate($strDbPath);
$oRemapHash{&MANIFEST_TARGET_PGDATA} = $strDbPath;
delete($oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'});
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, \%oManifest, \%oRemapHash, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'no tablespace remap - error when tablespace dir does not exist', ERROR_PATH_MISSING,
'--log-level-console=detail --tablespace-map-all=../../tablespace', false);
filePathCreate($oHostDbMaster->dbBasePath(2) . '/tablespace');
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
'no tablespace remap', undef, '--tablespace-map-all=../../tablespace --log-level-console=detail', false);
# Backup Info (with an empty stanza)
#-----------------------------------------------------------------------------------------------------------------------
executeTest('sudo chmod g+w ' . $oHostBackup->repoPath() . '/backup');
filePathCreate($oHostBackup->repoPath() . '/backup/db_empty', '0770');
$oHostBackup->info('normal output');
$oHostDbMaster->info('normal output', {strOutput => INFO_OUTPUT_JSON});
$oHostBackup->info('bogus stanza', {strStanza => 'bogus'});
$oHostDbMaster->info('bogus stanza', {strStanza => 'bogus', strOutput => INFO_OUTPUT_JSON});
# Dump out history path at the end to verify all history files are being recorded. This test is only performed locally
# because for some reason sort order is different when this command is executed via ssh (even though the content of the
# directory is identical).
#-----------------------------------------------------------------------------------------------------------------------
if ($bNeutralTest && !$bRemote)
{
executeTest('ls -1R ' . $oHostBackup->repoPath() . "/backup/${strStanza}/" . PATH_BACKUP_HISTORY,
{oLogTest => $oLogTest, bRemote => $bRemote});
}
# Test protocol shutdown timeout
#-----------------------------------------------------------------------------------------------------------------------
if ($bNeutralTest && $bRemote)
{
$oHostBackup->backup(
$strType, 'protocol shutdown timeout',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--protocol-timeout=2 --db-timeout=.5 --log-level-console=warn',
strTest => TEST_BACKUP_STOP, fTestDelay => 2, bSupplemental => false});
}
}
}
}
testCleanup(\$oLogTest);
}
#-------------------------------------------------------------------------------------------------------------------------------
# Test full
#
# Check the entire backup mechanism using actual clusters. Only the archive and start/stop mechanisms need to be tested since
# everything else was tested in the backup test.
#-------------------------------------------------------------------------------------------------------------------------------
if ($strTest eq 'all' || $strTest eq 'full')
{
$strThisTest = 'full';
$iRun = 0;
if (!$bVmOut)
{
&log(INFO, "Test Full Backup\n");
}
foreach my $bHostBackup (false, true)
{
foreach my $bHostStandby (false, true)
{
foreach my $strBackupDestination (
$bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER))
{
foreach my $bArchiveAsync ($bHostStandby ? (false) : (false, true))
{
foreach my $bCompress ($bHostStandby ? (false) : (false, true))
{
# Increment the run, log, and decide whether this unit test should be run
my $bLog = $iProcessMax == 1 && $oHostGroup->paramGet(HOST_PARAM_DB_VERSION) eq PG_VERSION_95;
next if (!testRun(
++$iRun,
"bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, asy ${bArchiveAsync}, cmp ${bCompress}",
$bLog ? $strModule : undef,
$bLog ? $strThisTest: undef,
\$oLogTest));
if ($bHostStandby && $oHostGroup->paramGet(HOST_PARAM_DB_VERSION) < PG_VERSION_HOT_STANDBY)
{
&log(INFO, 'skipped - this version of PostgreSQL does not support hot standby');
next;
}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = backupTestSetup(
false, $oLogTest,
{bHostBackup => $bHostBackup, bStandby => $bHostStandby, strBackupDestination => $strBackupDestination,
bCompress => $bCompress, bArchiveAsync => $bArchiveAsync});
# Determine if extra tests are performed. Extra tests should not be primary tests for compression or async archiving.
my $bTestExtra = ($iRun == 1) || ($iRun == 7);
# For the 'fail on missing archive.info file' test, the archive.info file must not be found so set archive invalid.
$oHostDbMaster->clusterCreate({bArchiveInvalid => $bTestExtra});
# Static backup parameters
my $fTestDelay = 1;
# Variable backup parameters
my $bDelta = true;
my $bForce = false;
my $strType = undef;
my $strTarget = undef;
my $bTargetExclusive = false;
my $strTargetAction;
my $strTargetTimeline = undef;
my $oRecoveryHashRef = undef;
my $strComment = undef;
my $iExpectedExitStatus = undef;
# Restore test string
my $strDefaultMessage = 'default';
my $strFullMessage = 'full';
my $strStandbyMessage = 'standby';
my $strIncrMessage = 'incr';
my $strTimeMessage = 'time';
my $strXidMessage = 'xid';
my $strNameMessage = 'name';
my $strTimelineMessage = 'timeline3';
# Create two new databases
$oHostDbMaster->sqlExecute('create database test1', {bAutoCommit => true});
$oHostDbMaster->sqlExecute('create database test2', {bAutoCommit => true});
# Test check command and stanza create
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra)
{
$strType = BACKUP_TYPE_FULL;
# NOTE: This must run before the success test since that will create the archive.info file
$oHostDbMaster->check(
'fail on missing archive.info file',
{iTimeout => 0.1, iExpectedExitStatus => ERROR_FILE_MISSING});
# Check ERROR_ARCHIVE_DISABLED error
$strComment = 'fail on archive_mode=off';
$oHostDbMaster->clusterRestart({bIgnoreLogError => true, bArchiveEnabled => false});
$oHostBackup->backup($strType, $strComment, {iExpectedExitStatus => ERROR_ARCHIVE_DISABLED});
$oHostDbMaster->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_DISABLED});
# If running the remote tests then also need to run check locally
if ($bHostBackup)
{
$oHostBackup->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_DISABLED});
}
# Check ERROR_ARCHIVE_COMMAND_INVALID error
$strComment = 'fail on invalid archive_command';
$oHostDbMaster->clusterRestart({bIgnoreLogError => true, bArchive => false});
$oHostBackup->backup($strType, $strComment, {iExpectedExitStatus => ERROR_ARCHIVE_COMMAND_INVALID});
$oHostDbMaster->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_COMMAND_INVALID});
# If running the remote tests then also need to run check locally
if ($bHostBackup)
{
$oHostBackup->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_COMMAND_INVALID});
}
# When archive-check=n then ERROR_FILE_MISSING will be raised instead of ERROR_ARCHIVE_COMMAND_INVALID
$strComment = 'fail on file missing when archive-check=n';
$oHostDbMaster->check(
$strComment,
{iTimeout => 0.1, iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-archive-check'});
# Stop the cluster ignoring any errors in the postgresql log
$oHostDbMaster->clusterStop({bIgnoreLogError => true});
# Providing a sufficient archive-timeout, verify that the check command runs successfully.
$strComment = 'verify success';
$oHostDbMaster->clusterStart();
$oHostDbMaster->check($strComment, {iTimeout => 5});
# If running the remote tests then also need to run check locally
if ($bHostBackup)
{
$oHostBackup->check($strComment, {iTimeout => 5});
}
# Check archive mismatch due to upgrade error
$strComment = 'fail on archive mismatch after upgrade';
# load the archive info file and munge it for testing by breaking the database version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
$oHostDbMaster->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH});
# If running the remote tests then also need to run check locally
if ($bHostBackup)
{
$oHostBackup->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH});
}
# Restore the file to its original condition
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
# Check archive_timeout error when WAL segment is not found
$strComment = 'fail on archive timeout';
$oHostDbMaster->clusterRestart({bIgnoreLogError => true, bArchiveInvalid => true});
$oHostDbMaster->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_TIMEOUT});
# If running the remote tests then also need to run check locally
if ($bHostBackup)
{
$oHostBackup->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_TIMEOUT});
}
# Restart the cluster ignoring any errors in the postgresql log
$oHostDbMaster->clusterRestart({bIgnoreLogError => true});
# With a valid archive info, create the backup.info file by running a backup then munge the backup.info file.
# Check backup mismatch error
$strComment = 'fail on backup info mismatch';
# First run a successful backup to create the backup.info file
$oHostBackup->backup($strType, 'run a successful backup');
# Load the backup.info file and munge it for testing by breaking the database version and system id
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB =>
{&INFO_BACKUP_KEY_DB_VERSION => '8.0', &INFO_BACKUP_KEY_SYSTEM_ID => 6999999999999999999}});
# Run the test
$oHostDbMaster->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_BACKUP_MISMATCH});
# If running the remote tests then also need to run check locally
if ($bHostBackup)
{
$oHostBackup->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_BACKUP_MISMATCH});
}
# Restore the file to its original condition
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
# Providing a sufficient archive-timeout, verify that the check command runs successfully now with valid
# archive.info and backup.info files
$strComment = 'verify success after backup';
$oHostDbMaster->check($strComment, {iTimeout => 5});
# If running the remote tests then also need to run check locally
if ($bHostBackup)
{
$oHostBackup->check($strComment, {iTimeout => 5});
}
# Restart the cluster ignoring any errors in the postgresql log
$oHostDbMaster->clusterRestart({bIgnoreLogError => true});
# Stanza Create
#--------------------------------------------------------------------------------
if ($bHostBackup)
{
# With data existing in the backup and archive directories, remove the info files
# Remove the archive.info file
executeTest('sudo rm -rf ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
$oHostBackup->stanzaCreate('fail on archive info file missing from non-empty dir',
{iExpectedExitStatus => ERROR_ARCHIVE_DIR_INVALID});
# Remove the backup.info file
executeTest('sudo rm -rf ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
$oHostBackup->stanzaCreate('fail on backup info file missing from non-empty dir',
{iExpectedExitStatus => ERROR_BACKUP_DIR_INVALID});
# Remove the repo sub-directories to ensure they do not exist
executeTest('sudo rm -rf ' . $oHostBackup->repoPath() . "/*");
$oHostBackup->stanzaCreate('verify success', {iTimeout => 5});
}
}
# Full backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_FULL;
# Create the table where test messages will be stored
$oHostDbMaster->sqlExecute("create table test (message text not null)");
$oHostDbMaster->sqlXlogRotate();
$oHostDbMaster->sqlExecute("insert into test values ('$strDefaultMessage')");
if ($bTestExtra)
{
# Acquire the backup advisory lock so it looks like a backup is running
if (!$oHostDbMaster->sqlSelectOne('select pg_try_advisory_lock(' . DB_BACKUP_ADVISORY_LOCK . ')'))
{
confess 'unable to acquire advisory lock for testing';
}
$oHostBackup->backup($strType, 'fail on backup lock exists', {iExpectedExitStatus => ERROR_LOCK_ACQUIRE});
# Release the backup advisory lock so the next backup will succeed
if (!$oHostDbMaster->sqlSelectOne('select pg_advisory_unlock(' . DB_BACKUP_ADVISORY_LOCK . ')'))
{
confess 'unable to release advisory lock';
}
}
my $oExecuteBackup = $oHostBackup->backupBegin(
$strType, 'update during backup',
{strTest => TEST_MANIFEST_BUILD, fTestDelay => $fTestDelay});
$oHostDbMaster->sqlExecute("update test set message = '$strFullMessage'");
# Required to set hint bits to be sent to the standby to make the heap match on both sides
$oHostDbMaster->sqlSelectOneTest('select message from test', $strFullMessage);
my $strFullBackup = $oHostBackup->backupEnd($strType, $oExecuteBackup);
# Kick out a bunch of archive logs to excercise async archiving. Only do this when compressed and remote to slow it
# down enough to make it evident that the async process is working.
if ($bArchiveAsync && $bCompress && $strBackupDestination eq HOST_BACKUP)
{
&log(INFO, ' multiple pg_switch_xlog() to exercise async archiving');
$oHostDbMaster->sqlExecute("create table xlog_activity (id int)");
$oHostDbMaster->sqlXlogRotate();
$oHostDbMaster->sqlExecute("insert into xlog_activity values (1)");
$oHostDbMaster->sqlXlogRotate();
$oHostDbMaster->sqlExecute("insert into xlog_activity values (2)");
$oHostDbMaster->sqlXlogRotate();
$oHostDbMaster->sqlExecute("insert into xlog_activity values (3)");
$oHostDbMaster->sqlXlogRotate();
$oHostDbMaster->sqlExecute("insert into xlog_activity values (4)");
$oHostDbMaster->sqlXlogRotate();
}
# Setup replica
#-----------------------------------------------------------------------------------------------------------------------
if ($bHostStandby)
{
$bDelta = false;
$bForce = false;
$strType = RECOVERY_TYPE_DEFAULT;
$strTarget = undef;
$bTargetExclusive = undef;
$strTargetAction = undef;
$strTargetTimeline = undef;
$oRecoveryHashRef = undef;
$strComment = undef;
$iExpectedExitStatus = undef;
$strComment = 'restore backup on replica';
my %oRemapHash;
$oRemapHash{&MANIFEST_TARGET_PGDATA} = $oHostDbStandby->dbBasePath();
if ($oHostDbStandby->dbVersion() >= PG_VERSION_92)
{
$oHostDbStandby->linkRemap(DB_PATH_PGXLOG, $oHostDbStandby->dbPath() . '/' . DB_PATH_PGXLOG);
}
$oHostDbStandby->restore(
OPTION_DEFAULT_RESTORE_SET, undef, \%oRemapHash, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive,
$strTargetAction, $strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus,
' --recovery-option=standby_mode=on' .
' --recovery-option="primary_conninfo=host=' . HOST_DB_MASTER . ' port=' . HOST_DB_PORT . ' user=replicator"');
$oHostDbStandby->clusterStart({bHotStandby => true});
# Make sure streaming replication is on
$oHostDbMaster->sqlSelectOneTest(
"select client_addr || '-' || state from pg_stat_replication", $oHostDbStandby->ipGet() . '/32-streaming');
# Check that the cluster was restored properly
$oHostDbStandby->sqlSelectOneTest('select message from test', $strFullMessage);
# Update message for standby
$oHostDbMaster->sqlExecute("update test set message = '$strStandbyMessage'");
my $strStandbyBackup = $oHostBackup->backup(
BACKUP_TYPE_FULL, 'backup from standby',
{bStandby => true,
iExpectedExitStatus => $oHostDbStandby->dbVersion() >= PG_VERSION_BACKUP_STANDBY ? undef : ERROR_CONFIG,
strOptionalParam => '--' . OPTION_RETENTION_FULL . '=1'});
if ($oHostDbStandby->dbVersion() >= PG_VERSION_BACKUP_STANDBY)
{
$strFullBackup = $strStandbyBackup;
}
}
# Execute stop and make sure the backup fails
#-----------------------------------------------------------------------------------------------------------------------
# Restart the cluster to check for any errors before continuing since the stop tests will definitely create errors and
# the logs will to be deleted to avoid causing issues further down the line.
if ($bTestExtra)
{
$strType = BACKUP_TYPE_INCR;
$oHostDbMaster->clusterRestart();
$oHostDbMaster->stop();
$oHostBackup->backup($strType, 'attempt backup when stopped', {iExpectedExitStatus => ERROR_STOP});
$oHostDbMaster->start();
}
# Setup the time target
#-----------------------------------------------------------------------------------------------------------------------
$oHostDbMaster->sqlExecute("update test set message = '$strTimeMessage'");
$oHostDbMaster->sqlXlogRotate();
my $strTimeTarget = $oHostDbMaster->sqlSelectOne("select to_char(current_timestamp, 'YYYY-MM-DD HH24:MI:SS.US TZ')");
&log(INFO, " time target is ${strTimeTarget}");
# Incr backup - fail on archive_mode=always when version >= 9.5
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra && $oHostDbMaster->dbVersion() >= PG_VERSION_95)
{
$strType = BACKUP_TYPE_INCR;
# Set archive_mode=always
$oHostDbMaster->clusterRestart({bArchiveAlways => true});
$oHostBackup->backup($strType, 'fail on archive_mode=always', {iExpectedExitStatus => ERROR_FEATURE_NOT_SUPPORTED});
# Reset the cluster to a normal state so the next test will work
$oHostDbMaster->clusterRestart();
}
# Incr backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
# Create a tablespace directory
filePathCreate($oHostDbMaster->tablespacePath(1), undef, undef, true);
# Also create it on the standby so replay won't fail
if (defined($oHostDbStandby))
{
filePathCreate($oHostDbStandby->tablespacePath(1), undef, undef, true);
}
$oHostDbMaster->sqlExecute(
"create tablespace ts1 location '" . $oHostDbMaster->tablespacePath(1) . "'", {bAutoCommit => true});
$oHostDbMaster->sqlExecute("alter table test set tablespace ts1", {bCheckPoint => true});
# Create a table in the tablespace
$oHostDbMaster->sqlExecute("create table test_remove (id int)");
$oHostDbMaster->sqlXlogRotate();
$oHostDbMaster->sqlExecute("update test set message = '$strDefaultMessage'");
$oHostDbMaster->sqlXlogRotate();
# Start a backup so the next backup has to restart it. This test is not required for PostgreSQL >= 9.6 since backups
# are run in non-exlusive mode.
if ($bTestExtra && $oHostDbMaster->dbVersion() >= PG_VERSION_93 && $oHostDbMaster->dbVersion() < PG_VERSION_96)
{
$oHostDbMaster->sqlSelectOne("select pg_start_backup('test backup that will cause an error', true)");
# Verify that an error is returned if the backup is already running
$oHostBackup->backup($strType, 'fail on backup already running', {iExpectedExitStatus => ERROR_DB_QUERY});
# Restart the cluster ignoring any errors in the postgresql log
$oHostDbMaster->clusterRestart({bIgnoreLogError => true});
# Start a new backup to make the next test restart it
$oHostDbMaster->sqlSelectOne("select pg_start_backup('test backup that will be restarted', true)");
}
$oExecuteBackup = $oHostBackup->backupBegin(
$strType, 'update during backup',
{strTest => TEST_MANIFEST_BUILD, fTestDelay => $fTestDelay,
strOptionalParam => '--' . OPTION_STOP_AUTO . ' --no-' . OPTION_BACKUP_ARCHIVE_CHECK});
# Drop a table
$oHostDbMaster->sqlExecute('drop table test_remove');
$oHostDbMaster->sqlXlogRotate();
$oHostDbMaster->sqlExecute("update test set message = '$strIncrMessage'", {bCommit => true});
# Check that application name is set
if ($oHostDbMaster->dbVersion() >= PG_VERSION_APPLICATION_NAME)
{
my $strApplicationNameExpected = BACKREST_NAME . ' [' . CMD_BACKUP . ']';
my $strApplicationName = $oHostDbMaster->sqlSelectOne(
"select application_name from pg_stat_activity where application_name like '" . BACKREST_NAME . "%'");
if (!defined($strApplicationName) || $strApplicationName ne $strApplicationNameExpected)
{
confess &log(ERROR,
"application name '" . (defined($strApplicationName) ? $strApplicationName : '[null]') .
"' does not match '" . $strApplicationNameExpected . "'");
}
}
my $strIncrBackup = $oHostBackup->backupEnd($strType, $oExecuteBackup);
# Setup the xid target
#-----------------------------------------------------------------------------------------------------------------------
my $strXidTarget = undef;
if ($bTestExtra)
{
$oHostDbMaster->sqlExecute("update test set message = '$strXidMessage'", {bCommit => false});
$oHostDbMaster->sqlXlogRotate();
$strXidTarget = $oHostDbMaster->sqlSelectOne("select txid_current()");
$oHostDbMaster->sqlCommit();
&log(INFO, " xid target is ${strXidTarget}");
}
# Setup the name target
#-----------------------------------------------------------------------------------------------------------------------
my $strNameTarget = 'backrest';
if ($bTestExtra)
{
$oHostDbMaster->sqlExecute("update test set message = '$strNameMessage'", {bCommit => true});
$oHostDbMaster->sqlXlogRotate();
if ($oHostDbMaster->dbVersion() >= PG_VERSION_91)
{
$oHostDbMaster->sqlExecute("select pg_create_restore_point('${strNameTarget}')");
}
&log(INFO, " name target is ${strNameTarget}");
}
# Create a table and data in database test2
#-----------------------------------------------------------------------------------------------------------------------
$oHostDbMaster->sqlExecute(
'create table test (id int);' .
'insert into test values (1);' .
'create table test_ts1 (id int) tablespace ts1;' .
'insert into test_ts1 values (2);',
{strDb => 'test2', bAutoCommit => true});
$oHostDbMaster->sqlXlogRotate();
# Restore (type = default)
#-----------------------------------------------------------------------------------------------------------------------
$bDelta = false;
$bForce = false;
$strType = RECOVERY_TYPE_DEFAULT;
$strTarget = undef;
$bTargetExclusive = undef;
$strTargetAction = undef;
$strTargetTimeline = undef;
$oRecoveryHashRef = undef;
$strComment = undef;
$iExpectedExitStatus = undef;
# &log(INFO, " testing recovery type = ${strType}");
if ($bTestExtra)
{
# Expect failure because postmaster.pid exists
$strComment = 'postmaster running';
$iExpectedExitStatus = ERROR_POSTMASTER_RUNNING;
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive,
$strTargetAction, $strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus);
}
$oHostDbMaster->clusterStop();
if ($bTestExtra)
{
# Expect failure because db path is not empty
$strComment = 'path not empty';
$iExpectedExitStatus = ERROR_RESTORE_PATH_NOT_EMPTY;
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive,
$strTargetAction, $strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus);
}
# Drop and recreate db path
testPathRemove($oHostDbMaster->dbBasePath());
filePathCreate($oHostDbMaster->dbBasePath());
testPathRemove($oHostDbMaster->dbPath() . '/pg_xlog');
filePathCreate($oHostDbMaster->dbPath() . '/pg_xlog');
testPathRemove($oHostDbMaster->tablespacePath(1));
filePathCreate($oHostDbMaster->tablespacePath(1));
# Now the restore should work
$strComment = undef;
$iExpectedExitStatus = undef;
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive,
$strTargetAction, $strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus, ' --db-include=test1');
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $bTestExtra ? $strNameMessage : $strIncrMessage);
# Now it should be OK to drop database test2
$oHostDbMaster->sqlExecute('drop database test2', {bAutoCommit => true});
# The test table lives in ts1 so it needs to be moved or dropped
if ($oHostDbMaster->dbVersion() >= PG_VERSION_90)
{
$oHostDbMaster->sqlExecute('alter table test set tablespace pg_default');
}
# Drop for older versions
else
{
$oHostDbMaster->sqlExecute('drop table test');
}
# And drop the tablespace
$oHostDbMaster->sqlExecute("drop tablespace ts1", {bAutoCommit => true});
# Restore (restore type = immediate, inclusive)
#-----------------------------------------------------------------------------------------------------------------------
if (($bTestExtra || $bHostStandby) && $oHostDbMaster->dbVersion() >= PG_VERSION_94)
{
$bDelta = false;
$bForce = true;
$strType = RECOVERY_TYPE_IMMEDIATE;
$strTarget = undef;
$bTargetExclusive = undef;
$strTargetAction = undef;
$strTargetTimeline = undef;
$oRecoveryHashRef = undef;
$strComment = undef;
$iExpectedExitStatus = undef;
&log(INFO, " testing recovery type = ${strType}");
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
$strFullBackup, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive, $strTargetAction,
$strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus, undef);
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest(
'select message from test', ($bHostStandby ? $strStandbyMessage : $strFullMessage));
}
# Restore (restore type = xid, inclusive)
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra)
{
$bDelta = false;
$bForce = true;
$strType = RECOVERY_TYPE_XID;
$strTarget = $strXidTarget;
$bTargetExclusive = undef;
$strTargetAction = $oHostDbMaster->dbVersion() >= PG_VERSION_91 ? 'promote' : undef;
$strTargetTimeline = undef;
$oRecoveryHashRef = undef;
$strComment = undef;
$iExpectedExitStatus = undef;
&log(INFO, " testing recovery type = ${strType}");
$oHostDbMaster->clusterStop();
executeTest('rm -rf ' . $oHostDbMaster->dbBasePath() . "/*");
executeTest('rm -rf ' . $oHostDbMaster->dbPath() . "/pg_xlog/*");
$oHostDbMaster->restore(
$strIncrBackup, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive, $strTargetAction,
$strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus,
'--tablespace-map-all=../../tablespace', false);
# Save recovery file to test so we can use it in the next test
$oFile->copy(PATH_ABSOLUTE, $oHostDbMaster->dbBasePath() . '/recovery.conf',
PATH_ABSOLUTE, "${strTestPath}/recovery.conf");
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $strXidMessage);
$oHostDbMaster->sqlExecute("update test set message = '$strTimelineMessage'");
}
# Restore (restore type = preserve, inclusive)
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra)
{
$bDelta = false;
$bForce = false;
$strType = RECOVERY_TYPE_PRESERVE;
$strTarget = undef;
$bTargetExclusive = undef;
$strTargetAction = undef;
$strTargetTimeline = undef;
$oRecoveryHashRef = undef;
$strComment = undef;
$iExpectedExitStatus = undef;
&log(INFO, " testing recovery type = ${strType}");
$oHostDbMaster->clusterStop();
executeTest('rm -rf ' . $oHostDbMaster->dbBasePath() . "/*");
executeTest('rm -rf ' . $oHostDbMaster->dbPath() . "/pg_xlog/*");
executeTest('rm -rf ' . $oHostDbMaster->tablespacePath(1) . "/*");
# Restore recovery file that was saved in last test
$oFile->move(PATH_ABSOLUTE, "${strTestPath}/recovery.conf",
PATH_ABSOLUTE, $oHostDbMaster->dbBasePath() . '/recovery.conf');
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive,
$strTargetAction, $strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus);
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $strXidMessage);
$oHostDbMaster->sqlExecute("update test set message = '$strTimelineMessage'");
}
# Restore (restore type = time, inclusive) - there is no exclusive time test because I can't find a way to find the
# exact commit time of a transaction.
#-----------------------------------------------------------------------------------------------------------------------
$bDelta = true;
$bForce = false;
$strType = RECOVERY_TYPE_TIME;
$strTarget = $strTimeTarget;
$bTargetExclusive = undef;
$strTargetAction = undef;
$strTargetTimeline = undef;
$oRecoveryHashRef = undef;
$strComment = undef;
$iExpectedExitStatus = undef;
&log(INFO, " testing recovery type = ${strType}");
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
$strFullBackup, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive, $strTargetAction,
$strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus);
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $strTimeMessage);
# Restore (restore type = xid, exclusive)
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra)
{
$bDelta = true;
$bForce = false;
$strType = RECOVERY_TYPE_XID;
$strTarget = $strXidTarget;
$bTargetExclusive = true;
$strTargetAction = undef;
$strTargetTimeline = undef;
$oRecoveryHashRef = undef;
$strComment = undef;
$iExpectedExitStatus = undef;
&log(INFO, " testing recovery type = ${strType}");
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
$strIncrBackup, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive, $strTargetAction,
$strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus);
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $strIncrMessage);
}
# Restore (restore type = name)
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra && $oHostDbMaster->dbVersion() >= PG_VERSION_91)
{
$bDelta = true;
$bForce = true;
$strType = RECOVERY_TYPE_NAME;
$strTarget = $strNameTarget;
$bTargetExclusive = undef;
$strTargetAction = undef;
$strTargetTimeline = undef;
$oRecoveryHashRef = undef;
$strComment = undef;
$iExpectedExitStatus = undef;
&log(INFO, " testing recovery type = ${strType}");
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive,
$strTargetAction, $strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus);
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $strNameMessage);
}
# Restore (restore type = default, timeline = 3)
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra && $oHostDbMaster->dbVersion() >= PG_VERSION_84)
{
$bDelta = true;
$bForce = false;
$strType = RECOVERY_TYPE_DEFAULT;
$strTarget = undef;
$bTargetExclusive = undef;
$strTargetAction = undef;
$strTargetTimeline = 4;
$oRecoveryHashRef = $oHostDbMaster->dbVersion() >= PG_VERSION_90 ? {'standby-mode' => 'on'} : undef;
$strComment = undef;
$iExpectedExitStatus = undef;
&log(INFO, " testing recovery type = ${strType}");
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
$strIncrBackup, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive, $strTargetAction,
$strTargetTimeline, $oRecoveryHashRef, $strComment, $iExpectedExitStatus);
$oHostDbMaster->clusterStart({bHotStandby => true});
$oHostDbMaster->sqlSelectOneTest('select message from test', $strTimelineMessage, {iTimeout => 120});
}
# Incr backup - make sure a --no-online backup fails
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra)
{
$strType = BACKUP_TYPE_INCR;
$oHostBackup->backup(
$strType, 'fail on --no-' . OPTION_ONLINE,
{iExpectedExitStatus => ERROR_POSTMASTER_RUNNING, strOptionalParam => '--no-' . OPTION_ONLINE});
}
# Incr backup - allow --no-online backup to succeed with --force
#-----------------------------------------------------------------------------------------------------------------------
if ($bTestExtra)
{
$strType = BACKUP_TYPE_INCR;
$oHostBackup->backup(
$strType, 'succeed on --no-' . OPTION_ONLINE . ' with --' . OPTION_FORCE,
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
}
# Stop clusters to catch any errors in the postgres log
#-----------------------------------------------------------------------------------------------------------------------
$oHostDbMaster->clusterStop({bImmediate => true});
if (defined($oHostDbStandby))
{
$oHostDbStandby->clusterStop({bImmediate => true});
}
testCleanup(\$oLogTest);
}
}
}
}
}
testCleanup();
}
}
our @EXPORT = qw(backupTestRun);
1;