You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-10-30 23:37:45 +02:00
Moved test and env modules to new directories to avoid namespace conflicts with common tests.
This commit is contained in:
David Steele
165
test/lib/pgBackRestTest/Module/Archive/ArchiveGetTest.pm
Normal file
165
test/lib/pgBackRestTest/Module/Archive/ArchiveGetTest.pm
Normal file
@@ -0,0 +1,165 @@
|
||||
####################################################################################################################################
|
||||
# ArchiveGetTest.pm - Tests for archive-get command
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Archive::ArchiveGetTest;
|
||||
use parent 'pgBackRestTest::Env::HostEnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
|
||||
use pgBackRest::Archive::ArchiveInfo;
|
||||
use pgBackRest::BackupInfo;
|
||||
use pgBackRest::DbVersion;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Ini;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Manifest;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
my $strArchiveChecksum = '72b9da071c13957fb4ca31f05dbd5c644297c2f7';
|
||||
my $iArchiveMax = 3;
|
||||
my $strArchiveTestFile = $self->dataPath() . '/backup.wal2_' . WAL_VERSION_94 . '.bin';
|
||||
|
||||
for (my $bRemote = false; $bRemote <= true; $bRemote++)
|
||||
{
|
||||
for (my $bCompress = false; $bCompress <= true; $bCompress++)
|
||||
{
|
||||
for (my $bExists = false; $bExists <= true; $bExists++)
|
||||
{
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}")) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress});
|
||||
|
||||
# Create the xlog path
|
||||
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
filePathCreate($strXlogPath, undef, false, true);
|
||||
|
||||
# Create the test path for pg_control and copy pg_control for stanza-create
|
||||
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
|
||||
executeTest(
|
||||
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
|
||||
DB_FILE_PGCONTROL);
|
||||
|
||||
my $strCommand =
|
||||
$oHostDbMaster->backrestExe() .
|
||||
' --config=' . $oHostDbMaster->backrestConfig() .
|
||||
' --stanza=' . $self->stanza() . ' archive-get';
|
||||
|
||||
# Fail on missing archive info file
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " 000000010000000100000001 ${strXlogPath}/000000010000000100000001",
|
||||
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
|
||||
|
||||
# Create the archive info file
|
||||
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . OPTION_ONLINE});
|
||||
|
||||
if (defined($self->expect()))
|
||||
{
|
||||
$self->expect()->supplementalAdd($oFile->pathGet(PATH_BACKUP_ARCHIVE) . '/archive.info');
|
||||
}
|
||||
|
||||
if ($bExists)
|
||||
{
|
||||
# Loop through archive files
|
||||
my $strArchiveFile;
|
||||
|
||||
for (my $iArchiveNo = 1; $iArchiveNo <= $iArchiveMax; $iArchiveNo++)
|
||||
{
|
||||
# Construct the archive filename
|
||||
if ($iArchiveNo > 255)
|
||||
{
|
||||
confess 'backup total * archive total cannot be greater than 255';
|
||||
}
|
||||
|
||||
$strArchiveFile = uc(sprintf('0000000100000001%08x', $iArchiveNo));
|
||||
|
||||
&log(INFO, ' archive ' .sprintf('%02x', $iArchiveNo) .
|
||||
" - ${strArchiveFile}");
|
||||
|
||||
my $strSourceFile = "${strArchiveFile}-${strArchiveChecksum}";
|
||||
|
||||
if ($bCompress)
|
||||
{
|
||||
$strSourceFile .= '.gz';
|
||||
}
|
||||
|
||||
# Change the directory permissions to enable file creation
|
||||
executeTest('sudo chmod 770 ' . dirname($oFile->pathGet(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . "-1")));
|
||||
filePathCreate(
|
||||
dirname(
|
||||
$oFile->pathGet(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . "-1/${strSourceFile}")), '0770', true, true);
|
||||
|
||||
$oFile->copy(
|
||||
PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file $strArchiveTestFile
|
||||
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . # Destination file
|
||||
"-1/${strSourceFile}",
|
||||
false, # Source is not compressed
|
||||
$bCompress, # Destination compress based on test
|
||||
undef, undef, # Unused params
|
||||
'0660', # Mode
|
||||
true); # Create path if it does not exist
|
||||
|
||||
my $strDestinationFile = "${strXlogPath}/${strArchiveFile}";
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . ($bRemote && $iArchiveNo == 1 ? ' --cmd-ssh=/usr/bin/ssh' : '') .
|
||||
" ${strArchiveFile} ${strDestinationFile}",
|
||||
{oLogTest => $self->expect()});
|
||||
|
||||
# Check that the destination file exists
|
||||
if ($oFile->exists(PATH_DB_ABSOLUTE, $strDestinationFile))
|
||||
{
|
||||
if ($oFile->hash(PATH_DB_ABSOLUTE, $strDestinationFile) ne $strArchiveChecksum)
|
||||
{
|
||||
confess "archive file hash does not match ${strArchiveChecksum}";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
confess 'archive file is not in destination';
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
$oHostDbMaster->stop();
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " 000000090000000900000009 ${strXlogPath}/RECOVERYXLOG",
|
||||
{iExpectedExitStatus => ERROR_STOP, oLogTest => $self->expect()});
|
||||
|
||||
$oHostDbMaster->start();
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " 000000090000000900000009 ${strXlogPath}/RECOVERYXLOG",
|
||||
{iExpectedExitStatus => 1, oLogTest => $self->expect()});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
337
test/lib/pgBackRestTest/Module/Archive/ArchivePushTest.pm
Normal file
337
test/lib/pgBackRestTest/Module/Archive/ArchivePushTest.pm
Normal file
@@ -0,0 +1,337 @@
|
||||
####################################################################################################################################
|
||||
# ArchivePushTest.pm - Tests for archive-push command
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Archive::ArchivePushTest;
|
||||
use parent 'pgBackRestTest::Env::HostEnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
|
||||
use pgBackRest::Archive::ArchiveInfo;
|
||||
use pgBackRest::BackupInfo;
|
||||
use pgBackRest::DbVersion;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Ini;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Manifest;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# archiveCheck
|
||||
#
|
||||
# Check that a WAL segment is present in the repository.
|
||||
####################################################################################################################################
|
||||
sub archiveCheck
|
||||
{
|
||||
my $self = shift;
|
||||
my $oFile = shift;
|
||||
my $strArchiveFile = shift;
|
||||
my $strArchiveChecksum = shift;
|
||||
my $bCompress = shift;
|
||||
my $strSpoolPath = shift;
|
||||
|
||||
# Build the archive name to check for at the destination
|
||||
my $strArchiveCheck = PG_VERSION_94 . "-1/${strArchiveFile}-${strArchiveChecksum}";
|
||||
|
||||
if ($bCompress)
|
||||
{
|
||||
$strArchiveCheck .= '.gz';
|
||||
}
|
||||
|
||||
my $oWait = waitInit(5);
|
||||
my $bFound = false;
|
||||
|
||||
do
|
||||
{
|
||||
$bFound = $oFile->exists(PATH_BACKUP_ARCHIVE, $strArchiveCheck);
|
||||
}
|
||||
while (!$bFound && waitMore($oWait));
|
||||
|
||||
if (!$bFound)
|
||||
{
|
||||
confess 'unable to find ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveCheck);
|
||||
}
|
||||
|
||||
if (defined($strSpoolPath))
|
||||
{
|
||||
fileRemove("${strSpoolPath}/archive/" . $self->stanza() . "/out/${strArchiveFile}.ok");
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
my $strArchiveChecksum = '72b9da071c13957fb4ca31f05dbd5c644297c2f7';
|
||||
my $iArchiveMax = 3;
|
||||
|
||||
for (my $bRemote = false; $bRemote <= true; $bRemote++)
|
||||
{
|
||||
for (my $bCompress = false; $bCompress <= true; $bCompress++)
|
||||
{
|
||||
for (my $bArchiveAsync = false; $bArchiveAsync <= true; $bArchiveAsync++)
|
||||
{
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, arc_async ${bArchiveAsync}", $self->processMax() == 1)) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => $bArchiveAsync});
|
||||
|
||||
# Create the xlog path
|
||||
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
filePathCreate($strXlogPath, undef, false, true);
|
||||
|
||||
# Create the test path for pg_control
|
||||
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
|
||||
|
||||
# Copy pg_control for stanza-create
|
||||
executeTest(
|
||||
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
|
||||
DB_FILE_PGCONTROL);
|
||||
|
||||
my $strCommand =
|
||||
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() .
|
||||
($bArchiveAsync ? ' --log-level-console=detail' : '') . ' --stanza=db archive-push';
|
||||
|
||||
# Test missing archive.info file
|
||||
&log(INFO, ' test archive.info missing');
|
||||
my $strSourceFile1 = $self->walSegment(1, 1, 1);
|
||||
filePathCreate("${strXlogPath}/archive_status");
|
||||
my $strArchiveFile1 = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, $strSourceFile1);
|
||||
|
||||
$oHostDbMaster->executeSimple($strCommand . " ${strXlogPath}/${strSourceFile1} --archive-max-mb=24",
|
||||
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile1}.error") if $bArchiveAsync;
|
||||
|
||||
# Create the archive info file
|
||||
$oHostBackup->stanzaCreate('create required data for stanza',
|
||||
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
|
||||
|
||||
my @stryExpectedWAL;
|
||||
|
||||
# Loop through backups
|
||||
for (my $iBackup = 1; $iBackup <= 3; $iBackup++)
|
||||
{
|
||||
my $strArchiveFile;
|
||||
|
||||
# Loop through archive files
|
||||
for (my $iArchive = 1; $iArchive <= $iArchiveMax; $iArchive++)
|
||||
{
|
||||
my $strSourceFile;
|
||||
|
||||
# Construct the archive filename
|
||||
my $iArchiveNo = (($iBackup - 1) * $iArchiveMax + ($iArchive - 1)) + 1;
|
||||
|
||||
if ($iArchiveNo > 255)
|
||||
{
|
||||
confess 'backup total * archive total cannot be greater than 255';
|
||||
}
|
||||
|
||||
$strSourceFile = $self->walSegment(1, 1, $iArchiveNo);
|
||||
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 2, $strSourceFile);
|
||||
|
||||
&log(INFO, ' backup ' . sprintf('%02d', $iBackup) .
|
||||
', archive ' .sprintf('%02x', $iArchive) .
|
||||
" - ${strArchiveFile}");
|
||||
|
||||
my $strArchiveTmp = undef;
|
||||
|
||||
if ($iBackup == 1 && $iArchive == 2)
|
||||
{
|
||||
# Should succeed when temp file already exists
|
||||
&log(INFO, ' test archive when tmp file exists');
|
||||
|
||||
$strArchiveTmp =
|
||||
$oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
|
||||
substr($strSourceFile, 0, 16) . "/${strSourceFile}" .
|
||||
($bCompress ? ".$oFile->{strCompressExtension}" : '') . '.pgbackrest.tmp';
|
||||
|
||||
executeTest('sudo chmod 770 ' . dirname($strArchiveTmp));
|
||||
fileStringWrite($strArchiveTmp, 'JUNK');
|
||||
|
||||
if ($bRemote)
|
||||
{
|
||||
executeTest('sudo chown ' . $oHostBackup->userGet() . " ${strArchiveTmp}");
|
||||
}
|
||||
}
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . ($bRemote && $iBackup == $iArchive ? ' --cmd-ssh=/usr/bin/ssh' : '') .
|
||||
" ${strXlogPath}/${strSourceFile}",
|
||||
{oLogTest => $self->expect()});
|
||||
push(
|
||||
@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}" .
|
||||
($bCompress ? ".$oFile->{strCompressExtension}" : ''));
|
||||
|
||||
# Make sure the temp file no longer exists
|
||||
if (defined($strArchiveTmp))
|
||||
{
|
||||
my $oWait = waitInit(5);
|
||||
my $bFound = true;
|
||||
|
||||
do
|
||||
{
|
||||
$bFound = fileExists($strArchiveTmp);
|
||||
}
|
||||
while ($bFound && waitMore($oWait));
|
||||
|
||||
if ($bFound)
|
||||
{
|
||||
confess "${strArchiveTmp} should have been removed by archive command";
|
||||
}
|
||||
}
|
||||
|
||||
if ($iArchive == $iBackup)
|
||||
{
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok") if $bArchiveAsync;
|
||||
|
||||
&log(INFO, ' test db version mismatch error');
|
||||
|
||||
$oHostBackup->infoMunge(
|
||||
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
|
||||
|
||||
&log(INFO, ' test db system-id mismatch error');
|
||||
|
||||
$oHostBackup->infoMunge(
|
||||
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;;
|
||||
|
||||
# Restore the file to its original condition
|
||||
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
|
||||
|
||||
# Fail because the process was killed
|
||||
if ($iBackup == 1 && !$bCompress)
|
||||
{
|
||||
&log(INFO, ' test stop');
|
||||
|
||||
if ($bArchiveAsync)
|
||||
{
|
||||
my $oExecArchive =
|
||||
$oHostDbMaster->execute(
|
||||
$strCommand . ' --test --test-delay=5 --test-point=' .
|
||||
lc(TEST_ARCHIVE_PUSH_ASYNC_START) . "=y ${strSourceFile}",
|
||||
{oLogTest => $self->expect(), iExpectedExitStatus => ERROR_TERM});
|
||||
$oExecArchive->begin();
|
||||
$oExecArchive->end(TEST_ARCHIVE_PUSH_ASYNC_START);
|
||||
|
||||
$oHostDbMaster->stop({bForce => true});
|
||||
$oExecArchive->end();
|
||||
}
|
||||
else
|
||||
{
|
||||
$oHostDbMaster->stop({strStanza => $oHostDbMaster->stanza()});
|
||||
}
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
{oLogTest => $self->expect(), iExpectedExitStatus => ERROR_STOP});
|
||||
|
||||
$oHostDbMaster->start({strStanza => $bArchiveAsync ? undef : $self->stanza()});
|
||||
}
|
||||
|
||||
# Should succeed because checksum is the same
|
||||
&log(INFO, ' test archive duplicate ok');
|
||||
|
||||
$oHostDbMaster->executeSimple($strCommand . " ${strXlogPath}/${strSourceFile}", {oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok") if $bArchiveAsync;
|
||||
|
||||
# Now it should break on archive duplication (because checksum is different
|
||||
&log(INFO, " test archive duplicate error");
|
||||
|
||||
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, $strSourceFile);
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
|
||||
|
||||
# Test .partial archive
|
||||
&log(INFO, " test .partial archive");
|
||||
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 2, "${strSourceFile}.partial");
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " --no-" . OPTION_REPO_SYNC . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
{oLogTest => $self->expect()});
|
||||
$self->archiveCheck(
|
||||
$oFile, "${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
|
||||
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
|
||||
|
||||
push(
|
||||
@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}" .
|
||||
($bCompress ? ".$oFile->{strCompressExtension}" : ''));
|
||||
|
||||
# Test .partial archive duplicate
|
||||
&log(INFO, ' test .partial archive duplicate');
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}.partial", {oLogTest => $self->expect()});
|
||||
$self->archiveCheck(
|
||||
$oFile, "${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
|
||||
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
|
||||
|
||||
# Test .partial archive with different checksum
|
||||
&log(INFO, ' test .partial archive with different checksum');
|
||||
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, "${strSourceFile}.partial");
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
|
||||
}
|
||||
else
|
||||
{
|
||||
$self->archiveCheck(
|
||||
$oFile, $strSourceFile, $strArchiveChecksum, $bCompress,
|
||||
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oFile->list(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001')},
|
||||
'(' . join(', ', @stryExpectedWAL) . ')',
|
||||
'all WAL in archive', {iWaitSeconds => 5});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
if (defined($self->expect()))
|
||||
{
|
||||
sleep(1); # Ugly hack to ensure repo is stable before checking files - replace in new tests
|
||||
$self->expect()->supplementalAdd($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
776
test/lib/pgBackRestTest/Module/Archive/ArchivePushUnitTest.pm
Normal file
776
test/lib/pgBackRestTest/Module/Archive/ArchivePushUnitTest.pm
Normal file
@@ -0,0 +1,776 @@
|
||||
####################################################################################################################################
|
||||
# ArchivePushUnitTest.pm - Unit tests for ArchivePush and ArchivePush Async
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Archive::ArchivePushUnitTest;
|
||||
use parent 'pgBackRestTest::Env::HostEnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
use English '-no_match_vars';
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
use Storable qw(dclone);
|
||||
|
||||
use pgBackRest::Archive::ArchiveCommon;
|
||||
use pgBackRest::Archive::ArchivePush;
|
||||
use pgBackRest::Archive::ArchivePushAsync;
|
||||
use pgBackRest::Archive::ArchivePushFile;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Lock ;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::DbVersion;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Protocol::Common;
|
||||
use pgBackRest::Protocol::Protocol;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Env::Host::HostBackupTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# initModule
|
||||
####################################################################################################################################
|
||||
sub initModule
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
$self->{strDbPath} = $self->testPath() . '/db';
|
||||
$self->{strWalPath} = "$self->{strDbPath}/pg_xlog";
|
||||
$self->{strWalStatusPath} = "$self->{strWalPath}/archive_status";
|
||||
$self->{strWalHash} = "1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
|
||||
$self->{strRepoPath} = $self->testPath() . '/repo';
|
||||
$self->{strArchivePath} = "$self->{strRepoPath}/archive/" . $self->stanza();
|
||||
$self->{strSpoolPath} = "$self->{strArchivePath}/out";
|
||||
|
||||
# Create the local file object
|
||||
$self->{oFile} =
|
||||
new pgBackRest::File
|
||||
(
|
||||
$self->stanza(),
|
||||
$self->{strRepoPath},
|
||||
new pgBackRest::Protocol::Common
|
||||
(
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
HOST_PROTOCOL_TIMEOUT # Protocol timeout
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# initTest
|
||||
####################################################################################################################################
|
||||
sub initTest
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Create WAL path
|
||||
filePathCreate($self->{strWalStatusPath}, undef, true, true);
|
||||
|
||||
# Create archive info
|
||||
filePathCreate($self->{strArchivePath}, undef, true, true);
|
||||
|
||||
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false);
|
||||
$oArchiveInfo->create(PG_VERSION_94, WAL_VERSION_94_SYS_ID, true);
|
||||
|
||||
$self->{strArchiveId} = $oArchiveInfo->archiveId();
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
my $oOption = {};
|
||||
|
||||
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
|
||||
$self->optionSetTest($oOption, OPTION_DB_PATH, $self->{strDbPath});
|
||||
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->{strRepoPath});
|
||||
$self->optionSetTest($oOption, OPTION_LOG_PATH, $self->testPath());
|
||||
$self->optionBoolSetTest($oOption, OPTION_COMPRESS, false);
|
||||
|
||||
$self->optionSetTest($oOption, OPTION_DB_TIMEOUT, 5);
|
||||
$self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 6);
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_TIMEOUT, 3);
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("ArchivePushFile::archivePushCheck"))
|
||||
{
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalSegment = '000000010000000100000001';
|
||||
|
||||
$self->testResult(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
'(9.4-1, [undef], [undef])', "${strWalSegment} WAL not found");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalMajorPath = "$self->{strArchivePath}/9.4-1/" . substr($strWalSegment, 0, 16);
|
||||
my $strWalSegmentHash = "${strWalSegment}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
|
||||
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
|
||||
|
||||
filePathCreate($strWalMajorPath, undef, false, true);
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testResult(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
'(9.4-1, 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27,' .
|
||||
" WAL segment ${strWalSegment} already exists in the archive with the same checksum\n" .
|
||||
'HINT: this is valid in some recovery scenarios but may also indicate a problem.)',
|
||||
"${strWalSegment} WAL found");
|
||||
|
||||
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegmentHash = "${strWalSegment}-10be15a0ab8e1653dfab18c83180e74f1507cab1";
|
||||
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testException(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strWalSegment} already exists in the archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegment = "${strWalSegment}.partial";
|
||||
$strWalSegmentHash = "${strWalSegment}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
|
||||
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
|
||||
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testResult(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
'(9.4-1, 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27,' .
|
||||
" WAL segment ${strWalSegment} already exists in the archive with the same checksum\n" .
|
||||
'HINT: this is valid in some recovery scenarios but may also indicate a problem.)',
|
||||
"${strWalSegment} WAL found");
|
||||
|
||||
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegmentHash = "${strWalSegment}-10be15a0ab8e1653dfab18c83180e74f1507cab1";
|
||||
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testException(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strWalSegment} already exists in the archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID)},
|
||||
ERROR_ASSERT, "strFile is required in File->hash");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strHistoryFile = "00000001.history";
|
||||
|
||||
fileStringWrite("$self->{strArchivePath}/9.4-1/${strHistoryFile}");
|
||||
|
||||
$self->testResult(sub {archivePushCheck(
|
||||
$self->{oFile}, $strHistoryFile, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strHistoryFile}")},
|
||||
'(9.4-1, [undef], [undef])', "history file ${strHistoryFile} found");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("ArchivePushFile::archivePushFile"))
|
||||
{
|
||||
my $iWalTimeline = 1;
|
||||
my $iWalMajor = 1;
|
||||
my $iWalMinor = 1;
|
||||
|
||||
$self->optionSetTest($oOption, OPTION_BACKUP_HOST, 'localhost');
|
||||
$self->optionSetTest($oOption, OPTION_BACKUP_USER, $self->pgUser());
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
# Create the file object
|
||||
my $oRemoteFile = new pgBackRest::File(
|
||||
$self->stanza(), $self->{strRepoPath}, protocolGet(BACKUP, undef, {strBackRestBin => $self->backrestExe()}));
|
||||
|
||||
# Generate a normal segment
|
||||
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->testResult(
|
||||
sub {archivePushFile($oRemoteFile, $self->{strWalPath}, $strSegment, false, false)}, '[undef]',
|
||||
"${strSegment} WAL segment to remote");
|
||||
|
||||
$self->testResult(
|
||||
sub {archivePushFile($oRemoteFile, $self->{strWalPath}, $strSegment, false, false)},
|
||||
"WAL segment 000000010000000100000001 already exists in the archive with the same checksum\n" .
|
||||
'HINT: this is valid in some recovery scenarios but may also indicate a problem.',
|
||||
"${strSegment} WAL duplicate segment to remote");
|
||||
|
||||
# Destroy protocol object
|
||||
protocolDestroy();
|
||||
|
||||
$self->optionReset($oOption, OPTION_BACKUP_HOST);
|
||||
$self->optionReset($oOption, OPTION_BACKUP_USER);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("ArchivePush->readyList()"))
|
||||
{
|
||||
my $oPushAsync = new pgBackRest::Archive::ArchivePushAsync($self->{strWalPath}, $self->{strSpoolPath});
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
$oPushAsync->initServer();
|
||||
|
||||
my $iWalTimeline = 1;
|
||||
my $iWalMajor = 1;
|
||||
my $iWalMinor = 1;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.done');
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '()',
|
||||
'ignore files without .ready extension');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '(000000010000000100000002, 000000010000000100000003)',
|
||||
'.ready files are found');
|
||||
|
||||
fileStringWrite("$self->{strSpoolPath}/000000010000000100000002.ok");
|
||||
fileStringWrite("$self->{strSpoolPath}/000000010000000100000003.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '(000000010000000100000004)',
|
||||
'new .ready files are found and duplicates ignored');
|
||||
|
||||
fileStringWrite("$self->{strSpoolPath}/000000010000000100000004.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '()',
|
||||
'no new .ready files returns empty list');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$iWalTimeline++;
|
||||
$iWalMinor = 1;
|
||||
|
||||
fileStringWrite("$self->{strWalStatusPath}/00000002.history.ready");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '(00000002.history)',
|
||||
'history .ready file');
|
||||
|
||||
fileStringWrite("$self->{strSpoolPath}/00000002.history.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.00000028.backup.ready');
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '(000000020000000100000001.00000028.backup)',
|
||||
'backup .ready file');
|
||||
|
||||
fileStringWrite("$self->{strSpoolPath}/000000020000000100000001.00000028.backup.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
fileRemove("$self->{strWalStatusPath}/00000002.history.ready");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '()', 'remove 00000002.history.ok file');
|
||||
|
||||
$self->testResult(
|
||||
sub {fileExists("$self->{strWalStatusPath}/00000002.history.ready")}, false, '00000002.history.ok is removed');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("ArchivePush->dropList()"))
|
||||
{
|
||||
my $oPushAsync = new pgBackRest::Archive::ArchivePushAsync($self->{strWalPath}, $self->{strSpoolPath});
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_QUEUE_MAX, PG_WAL_SIZE * 4);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
my $iWalTimeline = 1;
|
||||
my $iWalMajor = 1;
|
||||
my $iWalMinor = 1;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->dropList($oPushAsync->readyList())}, '()',
|
||||
'WAL files not dropped');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_QUEUE_MAX, PG_WAL_SIZE * 2);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->dropList($oPushAsync->readyList())},
|
||||
'(000000010000000100000001, 000000010000000100000002, 000000010000000100000003)', 'WAL files that exceed queue max');
|
||||
|
||||
# Reset queue max
|
||||
$self->optionReset($oOption, OPTION_ARCHIVE_QUEUE_MAX);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("ArchivePushAsync->walStatusWrite() & ArchivePush->walStatus()"))
|
||||
{
|
||||
my $oPush = new pgBackRest::Archive::ArchivePush();
|
||||
|
||||
my $oPushAsync = new pgBackRest::Archive::ArchivePushAsync($self->{strWalPath}, $self->{strSpoolPath});
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
$oPushAsync->initServer();
|
||||
|
||||
my $iWalTimeline = 1;
|
||||
my $iWalMajor = 1;
|
||||
my $iWalMinor = 1;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
|
||||
$self->testResult(sub {$oPush->walStatus($self->{strSpoolPath}, $strSegment)}, 0, "${strSegment} WAL no status");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a normal ok
|
||||
$oPushAsync->walStatusWrite(WAL_STATUS_OK, $strSegment);
|
||||
|
||||
# Check status
|
||||
$self->testResult(sub {$oPush->walStatus($self->{strSpoolPath}, $strSegment)}, 1, "${strSegment} WAL ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a bogus warning ok (if content is present there must be two lines)
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
fileStringWrite("$self->{strSpoolPath}/${strSegment}.ok", "Test Warning");
|
||||
|
||||
# Check status
|
||||
$self->testException(
|
||||
sub {$oPush->walStatus($self->{strSpoolPath}, $strSegment)}, ERROR_ASSERT,
|
||||
"${strSegment}.ok content must have at least two lines:\nTest Warning");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a valid warning ok
|
||||
$oPushAsync->walStatusWrite(WAL_STATUS_OK, $strSegment, 0, 'Test Warning');
|
||||
|
||||
# Check status
|
||||
$self->testResult(sub {$oPush->walStatus($self->{strSpoolPath}, $strSegment)}, 1, "${strSegment} WAL warning ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate an invalid error
|
||||
$self->testException(
|
||||
sub {$oPushAsync->walStatusWrite(WAL_STATUS_ERROR, $strSegment)}, ERROR_ASSERT,
|
||||
"error status must have iCode and strMessage set");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate an invalid error
|
||||
$self->testException(
|
||||
sub {$oPushAsync->walStatusWrite(WAL_STATUS_ERROR, $strSegment, ERROR_ASSERT)}, ERROR_ASSERT,
|
||||
"strMessage must be set when iCode is set");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate an invalid error
|
||||
fileStringWrite("$self->{strSpoolPath}/${strSegment}.error");
|
||||
|
||||
# Check status (will error because there are now two status files)
|
||||
$self->testException(
|
||||
sub {$oPush->walStatus($self->{strSpoolPath}, $strSegment);}, ERROR_ASSERT,
|
||||
"multiple status files found in " . $self->testPath() . "/repo/archive/db/out for ${strSegment}:" .
|
||||
" ${strSegment}.error, ${strSegment}.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Remove the ok file
|
||||
fileRemove("$self->{strSpoolPath}/${strSegment}.ok");
|
||||
|
||||
# Check status
|
||||
$self->testException(
|
||||
sub {$oPush->walStatus($self->{strSpoolPath}, $strSegment);}, ERROR_ASSERT, "${strSegment}.error has no content");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a valid error
|
||||
$oPushAsync->walStatusWrite(
|
||||
WAL_STATUS_ERROR, $strSegment, ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strSegment} already exists in the archive");
|
||||
|
||||
# Check status
|
||||
$self->testException(sub {
|
||||
$oPush->walStatus($self->{strSpoolPath}, $strSegment)}, ERROR_ARCHIVE_DUPLICATE,
|
||||
"WAL segment ${strSegment} already exists in the archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Change the error file to an ok file
|
||||
fileMove("$self->{strSpoolPath}/${strSegment}.error", "$self->{strSpoolPath}/${strSegment}.ok");
|
||||
|
||||
# Check status
|
||||
$self->testResult(
|
||||
sub {$oPush->walStatus($self->{strSpoolPath}, $strSegment);}, 1,
|
||||
"${strSegment} WAL warning ok (converted from .error)");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a normal ok
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$oPushAsync->walStatusWrite(WAL_STATUS_OK, $strSegment);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
|
||||
# Check status
|
||||
$self->testResult(sub {$oPush->walStatus($self->{strSpoolPath}, $strSegment)}, 0, "${strSegment} WAL no status");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("ArchivePushAsync->process()"))
|
||||
{
|
||||
my $oPushAsync = new pgBackRest::Archive::ArchivePushAsync(
|
||||
$self->{strWalPath}, $self->{strSpoolPath}, $self->backrestExe());
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
$oPushAsync->initServer();
|
||||
|
||||
my $iWalTimeline = 1;
|
||||
my $iWalMajor = 1;
|
||||
my $iWalMinor = 1;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a normal segment
|
||||
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
# Generate an error (.ready file withough a corresponding WAL file)
|
||||
my $strSegmentError = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
fileStringWrite("$self->{strWalStatusPath}/$strSegmentError.ready");
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(2, 0, 1, 1)', "process ${strSegment}, ${strSegmentError}");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileList($self->{strSpoolPath})}, "(${strSegment}.ok, ${strSegmentError}.error)",
|
||||
"${strSegment} pushed, ${strSegmentError} errored");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileStringRead("$self->{strSpoolPath}/$strSegmentError.error")},
|
||||
ERROR_FILE_OPEN . "\nraised on local-1 host: unable to open $self->{strWalPath}/${strSegmentError}",
|
||||
"test ${strSegmentError}.error contents");
|
||||
|
||||
# Remove pushed WAL file
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Fix errored WAL file by providing a valid segment
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegmentError);
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "process ${strSegment}, ${strSegmentError}");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegmentError)}, "${strSegmentError}-$self->{strWalHash}",
|
||||
"${strSegmentError} WAL in archive");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "${strSegmentError}.ok", "${strSegmentError} pushed");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Remove previously errored WAL file
|
||||
$self->walRemove($self->{strWalPath}, $strSegmentError);
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "remove ${strSegmentError}.ready");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "${strSegmentError} removed");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Enable compression
|
||||
$self->optionBoolSetTest($oOption, OPTION_COMPRESS, true);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
# Create history file
|
||||
my $strHistoryFile = "00000001.history";
|
||||
|
||||
fileStringWrite("$self->{strWalPath}/${strHistoryFile}");
|
||||
fileStringWrite("$self->{strWalStatusPath}/$strHistoryFile.ready");
|
||||
|
||||
# Create backup file
|
||||
my $strBackupFile = "${strSegment}.00000028.backup";
|
||||
|
||||
fileStringWrite("$self->{strWalPath}/${strBackupFile}");
|
||||
fileStringWrite("$self->{strWalStatusPath}/$strBackupFile.ready");
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(2, 0, 2, 0)', "end processing ${strHistoryFile}, ${strBackupFile}");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileList($self->{strSpoolPath})}, "(${strHistoryFile}.ok, ${strBackupFile}.ok)",
|
||||
"${strHistoryFile}, ${strBackupFile} pushed");
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->{oFile}->exists(PATH_BACKUP_ARCHIVE, "$self->{strArchiveId}/${strHistoryFile}")}, true,
|
||||
"${strHistoryFile} in archive");
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->{oFile}->exists(PATH_BACKUP_ARCHIVE, "$self->{strArchiveId}/${strBackupFile}")}, true,
|
||||
"${strBackupFile} in archive");
|
||||
|
||||
# Remove history and backup files
|
||||
fileRemove("$self->{strWalPath}/${strHistoryFile}");
|
||||
fileRemove("$self->{strWalStatusPath}/$strHistoryFile.ready");
|
||||
fileRemove("$self->{strWalPath}/${strBackupFile}");
|
||||
fileRemove("$self->{strWalStatusPath}/$strBackupFile.ready");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a normal segment
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "processing ${strSegment}.gz");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
# Remove the WAL and process so the .ok file is removed
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "remove ${strSegment}.ready");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "${strSegment}.ok removed");
|
||||
|
||||
# Generate the same WAL again
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "processed duplicate ${strSegment}.gz");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "${strSegment}.ok", "${strSegment} pushed");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileStringRead("$self->{strSpoolPath}/${strSegment}.ok")},
|
||||
"0\nWAL segment ${strSegment} already exists in the archive with the same checksum\n" .
|
||||
'HINT: this is valid in some recovery scenarios but may also indicate a problem.',
|
||||
"${strSegment}.ok warning status");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
# Remove the WAL
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
|
||||
# Disable compression
|
||||
$self->optionBoolSetTest($oOption, OPTION_COMPRESS, false);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_QUEUE_MAX, PG_WAL_SIZE * 2);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
# Generate WAL to test queue limits
|
||||
my @strySegment =
|
||||
(
|
||||
$self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++),
|
||||
$self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++),
|
||||
$self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++)
|
||||
);
|
||||
|
||||
foreach my $strSegment (@strySegment)
|
||||
{
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
}
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(3, 3, 1, 0)', "process and drop files");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileList($self->{strSpoolPath})}, '(' . join('.ok, ', @strySegment) . '.ok)',
|
||||
join(', ', @strySegment) . " ok drop files written");
|
||||
|
||||
foreach my $strSegment (@strySegment)
|
||||
{
|
||||
$self->testResult(
|
||||
sub {fileStringRead("$self->{strSpoolPath}/${strSegment}.ok")},
|
||||
$strSegment eq $strySegment[0] ? '' :
|
||||
"0\ndropped WAL file ${strSegment} because archive queue exceeded " . optionGet(OPTION_ARCHIVE_QUEUE_MAX) .
|
||||
' bytes',
|
||||
"verify ${strSegment} status");
|
||||
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
}
|
||||
|
||||
$self->optionReset($oOption, OPTION_ARCHIVE_QUEUE_MAX);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "final process to remove ok files");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "ok files removed");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("ArchivePush->process()"))
|
||||
{
|
||||
my $oPush = new pgBackRest::Archive::ArchivePush($self->backrestExe());
|
||||
|
||||
my $iWalTimeline = 1;
|
||||
my $iWalMajor = 1;
|
||||
my $iWalMinor = 1;
|
||||
|
||||
my $iProcessId = $PID;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Set db-host to trick archive-push into thinking it is running on the backup server
|
||||
$self->optionSetTest($oOption, OPTION_DB_HOST, BOGUS);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$self->testException(sub {$oPush->process(undef)}, ERROR_HOST_INVALID, 'archive-push operation must run on db host');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Reset db-host
|
||||
$self->optionReset($oOption, OPTION_DB_HOST);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$self->testException(sub {$oPush->process(undef)}, ERROR_PARAM_REQUIRED, 'WAL file to push required');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->testResult(sub {$oPush->process("pg_xlog/${strSegment}")}, 0, "${strSegment} WAL pushed (with relative path)");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Set unrealistic queue max to make synchronous push drop a WAL
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_QUEUE_MAX, 0);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL dropped");
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
# Set more realistic queue max and allow segment to push
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_QUEUE_MAX, PG_WAL_SIZE * 4);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL pushed");
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
|
||||
# Reset queue max
|
||||
$self->optionReset($oOption, OPTION_ARCHIVE_QUEUE_MAX);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Enable async archiving
|
||||
$self->optionBoolSetTest($oOption, OPTION_ARCHIVE_ASYNC, true);
|
||||
$self->optionSetTest($oOption, OPTION_SPOOL_PATH, $self->{strRepoPath});
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
# Write an error file and verify that it doesn't error the first time around
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
filePathCreate($self->{strSpoolPath}, undef, undef, true);
|
||||
fileStringWrite("$self->{strSpoolPath}/${strSegment}.error", ERROR_ARCHIVE_TIMEOUT . "\ntest error");
|
||||
|
||||
$self->testException(
|
||||
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_ARCHIVE_TIMEOUT,
|
||||
"test error");
|
||||
|
||||
$self->testResult($oPush->{bConfessOnError}, true, "went through error loop");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
"${strSegment} WAL not in archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Write an OK file so the async process is not actually started
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
fileStringWrite("$self->{strSpoolPath}/${strSegment}.ok");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0,
|
||||
"${strSegment} WAL pushed async from synthetic ok file");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
"${strSegment} WAL not in archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL pushed async");
|
||||
exit if ($iProcessId != $PID);
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_TIMEOUT, 1);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$self->testException(
|
||||
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_ARCHIVE_TIMEOUT,
|
||||
"unable to push WAL ${strSegment} asynchronously after 1 second(s)");
|
||||
exit if ($iProcessId != $PID);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->optionSetTest($oOption, OPTION_BACKUP_HOST, BOGUS);
|
||||
$self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 60);
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_TIMEOUT, 5);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$self->testException(
|
||||
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_HOST_CONNECT,
|
||||
'remote process terminated on ' . BOGUS . ' host.*');
|
||||
exit if ($iProcessId != $PID);
|
||||
|
||||
# Disable async archiving
|
||||
$self->optionReset($oOption, OPTION_ARCHIVE_ASYNC);
|
||||
$self->optionReset($oOption, OPTION_SPOOL_PATH);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
118
test/lib/pgBackRestTest/Module/Archive/ArchiveStopTest.pm
Normal file
118
test/lib/pgBackRestTest/Module/Archive/ArchiveStopTest.pm
Normal file
@@ -0,0 +1,118 @@
|
||||
####################################################################################################################################
|
||||
# ArchiveStopTest.pm - Tests for archive-push command to make sure aync queue limits are implemented correctly
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Archive::ArchiveStopTest;
|
||||
use parent 'pgBackRestTest::Env::HostEnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
|
||||
use pgBackRest::Archive::ArchiveInfo;
|
||||
use pgBackRest::BackupInfo;
|
||||
use pgBackRest::DbVersion;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Ini;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Manifest;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
my $strArchiveTestFile = $self->dataPath() . '/backup.wal2_' . WAL_VERSION_94 . '.bin';
|
||||
|
||||
for (my $bRemote = false; $bRemote <= true; $bRemote++)
|
||||
{
|
||||
for (my $bCompress = false; $bCompress <= true; $bCompress++)
|
||||
{
|
||||
for (my $iError = 0; $iError <= $bRemote; $iError++)
|
||||
{
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version'))) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true});
|
||||
|
||||
# Create compression extension
|
||||
my $strCompressExt = $bCompress ? ".$oFile->{strCompressExtension}" : '';
|
||||
|
||||
# Create the xlog path
|
||||
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
filePathCreate($strXlogPath, undef, false, true);
|
||||
|
||||
# Create the test path for pg_control and copy pg_control for stanza-create
|
||||
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
|
||||
executeTest(
|
||||
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
|
||||
DB_FILE_PGCONTROL);
|
||||
|
||||
# Create the archive info file
|
||||
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . OPTION_ONLINE});
|
||||
|
||||
# Push a WAL segment
|
||||
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 1);
|
||||
|
||||
# Break the database version of the archive info file
|
||||
if ($iError == 0)
|
||||
{
|
||||
$oHostBackup->infoMunge(
|
||||
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
|
||||
}
|
||||
|
||||
# Push two more segments with errors to exceed archive-max-mb
|
||||
$oHostDbMaster->archivePush(
|
||||
$strXlogPath, $strArchiveTestFile, 2, $iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
|
||||
|
||||
$oHostDbMaster->archivePush(
|
||||
$strXlogPath, $strArchiveTestFile, 3, $iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
|
||||
|
||||
# Now this segment will get dropped
|
||||
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 4);
|
||||
|
||||
# Fix the database version
|
||||
if ($iError == 0)
|
||||
{
|
||||
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
|
||||
}
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oFile->list(
|
||||
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001', {strExpression => '^(?!000000010000000100000002).+'})},
|
||||
"000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt}",
|
||||
'segment 2-4 not pushed (2 is pushed sometimes when remote but ignore)', {iWaitSeconds => 5});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 5);
|
||||
|
||||
$self->testResult(
|
||||
sub {$oFile->list(
|
||||
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001', {strExpression => '^(?!000000010000000100000002).+'})},
|
||||
"(000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt}, " .
|
||||
"000000010000000100000005-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt})",
|
||||
'segment 5 is pushed', {iWaitSeconds => 5});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
185
test/lib/pgBackRestTest/Module/Archive/ArchiveUnitTest.pm
Normal file
185
test/lib/pgBackRestTest/Module/Archive/ArchiveUnitTest.pm
Normal file
@@ -0,0 +1,185 @@
|
||||
####################################################################################################################################
|
||||
# ArchiveUnitTest.pm - Tests for ArchiveCommon module
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Archive::ArchiveUnitTest;
|
||||
use parent 'pgBackRestTest::Env::HostEnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
|
||||
use pgBackRest::Archive::ArchiveCommon;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
|
||||
use pgBackRestTest::Env::Host::HostBackupTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
my $strModule = 'ArchiveCommon';
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("${strModule}::walPath()"))
|
||||
{
|
||||
my $strDbPath = '/db';
|
||||
my $strWalFileRelative = 'pg_xlog/000000010000000100000001';
|
||||
my $strWalFileAbsolute = "${strDbPath}/${strWalFileRelative}";
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {walPath($strWalFileRelative, undef, CMD_ARCHIVE_GET)}, ERROR_OPTION_REQUIRED,
|
||||
"option '" . OPTION_DB_PATH . "' must be specified when relative xlog paths are used\n" .
|
||||
"HINT: Is \%f passed to " . CMD_ARCHIVE_GET . " instead of \%p?\n" .
|
||||
"HINT: PostgreSQL may pass relative paths even with \%p depending on the environment.");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {walPath($strWalFileRelative, $strDbPath, CMD_ARCHIVE_PUSH)}, $strWalFileAbsolute, 'relative path is contructed');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {walPath($strWalFileAbsolute, $strDbPath, CMD_ARCHIVE_PUSH)}, $strWalFileAbsolute,
|
||||
'path is not relative and db-path is still specified');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {walPath($strWalFileAbsolute, $strDbPath, CMD_ARCHIVE_PUSH)}, $strWalFileAbsolute,
|
||||
'path is not relative and db-path is undef');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("${strModule}::walIsSegment()"))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {walIsSegment('0000000200ABCDEF0000001')}, false, 'invalid segment');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {walIsSegment('0000000200ABCDEF00000001')}, true, 'valid segment');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {walIsSegment('000000010000000100000001.partial')}, true, 'valid partial segment');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {walIsSegment('00000001.history')}, false, 'valid history file');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {walIsSegment('000000020000000100000001.00000028.backup')}, false, 'valid backup file');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("${strModule}::walIsPartial()"))
|
||||
{
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalSegment = '0000000200ABCDEF00000001';
|
||||
|
||||
$self->testResult(sub {walIsPartial($strWalSegment)}, false, "${strWalSegment} WAL is not partial");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegment = $strWalSegment . '.partial';
|
||||
|
||||
$self->testResult(sub {walIsPartial($strWalSegment)}, true, "${strWalSegment} WAL is partial");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("${strModule}::walSegmentFind()"))
|
||||
{
|
||||
my $strArchiveId = '9.4-1';
|
||||
my $oFile = new pgBackRest::File(
|
||||
$self->stanza(),
|
||||
$self->testPath(),
|
||||
new pgBackRest::Protocol::Common(
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
HOST_PROTOCOL_TIMEOUT # Protocol timeout
|
||||
));
|
||||
|
||||
my $strArchivePath = $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveId);;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalSegment = '000000010000000100000001ZZ';
|
||||
|
||||
$self->testException(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, ERROR_ASSERT, "${strWalSegment} is not a WAL segment");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegment = '000000010000000100000001';
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, undef, "${strWalSegment} WAL not found");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment, .1)}, ERROR_ARCHIVE_TIMEOUT,
|
||||
"could not find WAL segment ${strWalSegment} after 0.1 second(s)");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalMajorPath = "${strArchivePath}/" . substr($strWalSegment, 0, 16);
|
||||
my $strWalSegmentHash = "${strWalSegment}-53aa5d59515aa7288ae02ba414c009aed1ca73ad";
|
||||
|
||||
filePathCreate($strWalMajorPath, undef, false, true);
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, substr($strWalSegment, 8, 16))}, $strWalSegmentHash,
|
||||
"${strWalSegment} WAL found without timeline");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalSegmentHash2 = "${strWalSegment}-a0b0d38b8aa263e25b8ff52a0a4ba85b6be97f9b.gz";
|
||||
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash2}");
|
||||
|
||||
$self->testException(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, ERROR_ARCHIVE_DUPLICATE,
|
||||
"duplicates found in archive for WAL segment ${strWalSegment}: ${strWalSegmentHash}, ${strWalSegmentHash2}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalSegment3 = '00000002' . substr($strWalSegment, 8, 16);
|
||||
my $strWalSegmentHash3 = "${strWalSegment3}-dcdd09246e1918e88c67cf44b35edc23b803d879";
|
||||
my $strWalMajorPath3 = "${strArchivePath}/" . substr($strWalSegment3, 0, 16);
|
||||
|
||||
filePathCreate($strWalMajorPath3, undef, false, true);
|
||||
fileStringWrite("${strWalMajorPath3}/${strWalSegmentHash3}");
|
||||
|
||||
$self->testException(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, substr($strWalSegment, 8, 16))}, ERROR_ARCHIVE_DUPLICATE,
|
||||
"duplicates found in archive for WAL segment XXXXXXXX" . substr($strWalSegment, 8, 16) .
|
||||
": ${strWalSegmentHash}, ${strWalSegmentHash2}, ${strWalSegmentHash3}");
|
||||
|
||||
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
fileRemove("${strWalMajorPath3}/${strWalSegmentHash3}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash2,
|
||||
"${strWalSegment} WAL found with compressed extension");
|
||||
|
||||
fileRemove("${strWalMajorPath}/${strWalSegmentHash2}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegment = $strWalSegment . '.partial';
|
||||
$strWalSegmentHash = "${strWalSegment}-996195c807713ef9262170043e7222cb150aef70";
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
Reference in New Issue
Block a user