You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-06-14 23:44:58 +02:00
Consolidate archive-get test into new archive test.
This commit is contained in:
David Steele
@ -1,183 +0,0 @@
|
||||
####################################################################################################################################
|
||||
# ArchiveGetTest.pm - Tests for archive-get command
|
||||
####################################################################################################################################
|
||||
package pgBackRestTest::Module::Archive::ArchiveGetTest;
|
||||
use parent 'pgBackRestTest::Env::HostEnvTest';
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
|
||||
use pgBackRest::Archive::Info;
|
||||
use pgBackRest::Backup::Info;
|
||||
use pgBackRest::DbVersion;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Ini;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::FileTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
my $strArchiveChecksum = '72b9da071c13957fb4ca31f05dbd5c644297c2f7';
|
||||
my $iArchiveMax = 3;
|
||||
my $strArchiveTestFile = $self->dataPath() . '/backup.wal2_' . WAL_VERSION_94 . '.bin';
|
||||
|
||||
foreach my $bS3 (false, true)
|
||||
{
|
||||
foreach my $bRemote ($bS3 ? (true) : (false, true))
|
||||
{
|
||||
foreach my $bCompress ($bS3 ? (true) : (false, true))
|
||||
{
|
||||
foreach my $bExists ($bS3 ? (true) : (false, true))
|
||||
{
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}, s3 ${bS3}")) {next}
|
||||
|
||||
# Create hosts and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bS3 => $bS3});
|
||||
|
||||
# Create the xlog path
|
||||
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
storageDb()->pathCreate($strXlogPath, {bCreateParent => true});
|
||||
|
||||
# Create the test path for pg_control and copy pg_control for stanza-create
|
||||
storageDb()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
|
||||
executeTest(
|
||||
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
|
||||
DB_FILE_PGCONTROL);
|
||||
|
||||
my $strCommand =
|
||||
$oHostDbMaster->backrestExe() .
|
||||
' --config=' . $oHostDbMaster->backrestConfig() .
|
||||
' --stanza=' . $self->stanza() . ' archive-get';
|
||||
|
||||
# Fail on missing archive info file
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " 000000010000000100000001 ${strXlogPath}/000000010000000100000001",
|
||||
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
|
||||
|
||||
# Create the archive info file
|
||||
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . OPTION_ONLINE});
|
||||
|
||||
if (defined($self->expect()))
|
||||
{
|
||||
$self->expect()->supplementalAdd(
|
||||
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), undef,
|
||||
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
|
||||
}
|
||||
|
||||
if ($bExists)
|
||||
{
|
||||
# Loop through archive files
|
||||
my $strArchiveFile;
|
||||
|
||||
for (my $iArchiveNo = 1; $iArchiveNo <= $iArchiveMax; $iArchiveNo++)
|
||||
{
|
||||
# Construct the archive filename
|
||||
if ($iArchiveNo > 255)
|
||||
{
|
||||
confess 'backup total * archive total cannot be greater than 255';
|
||||
}
|
||||
|
||||
$strArchiveFile = uc(sprintf('0000000100000001%08x', $iArchiveNo));
|
||||
|
||||
&log(INFO, ' archive ' .sprintf('%02x', $iArchiveNo) .
|
||||
" - ${strArchiveFile}");
|
||||
|
||||
my $strSourceFile = "${strArchiveFile}-${strArchiveChecksum}";
|
||||
|
||||
if ($bCompress)
|
||||
{
|
||||
$strSourceFile .= '.gz';
|
||||
}
|
||||
|
||||
# Change the directory permissions to enable file creation
|
||||
forceStorageMode(storageRepo(), STORAGE_REPO_ARCHIVE, '770');
|
||||
|
||||
storageRepo()->pathCreate(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16),
|
||||
{strMode => '0770', bIgnoreExists => true, bCreateParent => true});
|
||||
|
||||
storageTest()->copy(
|
||||
$strArchiveTestFile,
|
||||
storageRepo()->openWrite(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . "-1/${strSourceFile}",
|
||||
{rhyFilter => $bCompress ? [{strClass => STORAGE_FILTER_GZIP}] : undef,
|
||||
strMode => '0660', bCreateParent => true}));
|
||||
|
||||
my ($strActualChecksum) = storageRepo()->hashSize(
|
||||
storageRepo()->openRead(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . "-1/${strSourceFile}",
|
||||
{rhyFilter => $bCompress ?
|
||||
[{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef}));
|
||||
|
||||
if ($strActualChecksum ne $strArchiveChecksum)
|
||||
{
|
||||
confess "archive file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
|
||||
}
|
||||
|
||||
my $strDestinationFile = "${strXlogPath}/${strArchiveFile}";
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . ($bRemote && $iArchiveNo == 1 ? ' --cmd-ssh=/usr/bin/ssh' : '') .
|
||||
" ${strArchiveFile} ${strDestinationFile}",
|
||||
{oLogTest => $self->expect()});
|
||||
|
||||
# Check that the destination file exists
|
||||
if (storageDb()->exists($strDestinationFile))
|
||||
{
|
||||
my ($strActualChecksum) = storageDb()->hashSize($strDestinationFile);
|
||||
|
||||
if ($strActualChecksum ne $strArchiveChecksum)
|
||||
{
|
||||
confess "recovered file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
confess "archive file '${strDestinationFile}' is not in destination";
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
$oHostDbMaster->stop();
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " 000000090000000900000009 ${strXlogPath}/RECOVERYXLOG",
|
||||
{iExpectedExitStatus => ERROR_STOP, oLogTest => $self->expect()});
|
||||
|
||||
$oHostDbMaster->start();
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " 000000090000000900000009 ${strXlogPath}/RECOVERYXLOG",
|
||||
{iExpectedExitStatus => 1, oLogTest => $self->expect()});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@ -92,6 +92,7 @@ sub run
|
||||
|
||||
# Reduce console logging to detail
|
||||
$oHostDbMaster->configUpdate({&CONFIG_SECTION_GLOBAL => {&OPTION_LOG_LEVEL_CONSOLE => lc(DETAIL)}});
|
||||
my $strLogDebug = '--' . OPTION_LOG_LEVEL_CONSOLE . qw{=} . lc(DEBUG);
|
||||
|
||||
# If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing
|
||||
# without slowing down the other tests too much.
|
||||
@ -100,8 +101,9 @@ sub run
|
||||
$oHostBackup->configUpdate({&CONFIG_SECTION_GLOBAL => {&OPTION_PROCESS_MAX => 2}});
|
||||
$oHostDbMaster->configUpdate({&CONFIG_SECTION_GLOBAL => {&OPTION_PROCESS_MAX => 2}});
|
||||
|
||||
# Reduce console logging to warn
|
||||
# Reduce console logging to warn (even for debug exceptions)
|
||||
$oHostDbMaster->configUpdate({&CONFIG_SECTION_GLOBAL => {&OPTION_LOG_LEVEL_CONSOLE => lc(WARN)}});
|
||||
$strLogDebug = '--' . OPTION_LOG_LEVEL_CONSOLE . qw{=} . lc(WARN);
|
||||
}
|
||||
|
||||
# Create the xlog path
|
||||
@ -117,8 +119,13 @@ sub run
|
||||
$oHostDbMaster->dbBasePath() . qw{/} . DB_FILE_PGCONTROL);
|
||||
|
||||
# Create archive-push command
|
||||
my $strCommand =
|
||||
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=db archive-push';
|
||||
my $strCommandPush =
|
||||
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=' . $self->stanza() .
|
||||
' ' . CMD_ARCHIVE_PUSH;
|
||||
|
||||
my $strCommandGet =
|
||||
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=' . $self->stanza() .
|
||||
' ' . CMD_ARCHIVE_GET;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
&log(INFO, ' archive.info missing');
|
||||
@ -126,7 +133,12 @@ sub run
|
||||
storageTest()->pathCreate("${strXlogPath}/archive_status");
|
||||
my $strArchiveFile1 = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, $strSourceFile1);
|
||||
|
||||
$oHostDbMaster->executeSimple($strCommand . " ${strXlogPath}/${strSourceFile1} --archive-max-mb=24",
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandPush . " ${strXlogPath}/${strSourceFile1}",
|
||||
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandGet . " ${strSourceFile1} ${strXlogPath}/RECOVERYXLOG",
|
||||
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@ -140,13 +152,38 @@ sub run
|
||||
my $strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 2, $strSourceFile);
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . " ${strXlogPath}/${strSourceFile}",
|
||||
$strCommandPush . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . " ${strLogDebug} ${strXlogPath}/${strSourceFile}",
|
||||
{oLogTest => $self->expect()});
|
||||
push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}");
|
||||
|
||||
# Test that the WAL was pushed
|
||||
$self->archiveCheck($strSourceFile, $strArchiveChecksum, false);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
&log(INFO, ' get first WAL');
|
||||
|
||||
# Remove WAL so it can be recovered
|
||||
storageTest()->remove("${strXlogPath}/${strSourceFile}", {bIgnoreMissing => false});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandGet . " ${strLogDebug} ${strSourceFile} ${strXlogPath}/RECOVERYXLOG",
|
||||
{oLogTest => $self->expect()});
|
||||
|
||||
# Check that the destination file exists
|
||||
if (storageDb()->exists("${strXlogPath}/RECOVERYXLOG"))
|
||||
{
|
||||
my ($strActualChecksum) = storageDb()->hashSize("${strXlogPath}/RECOVERYXLOG");
|
||||
|
||||
if ($strActualChecksum ne $strArchiveChecksum)
|
||||
{
|
||||
confess "recovered file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
confess "archive file '${strXlogPath}/RECOVERYXLOG' is not in destination";
|
||||
}
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
&log(INFO, ' push second WAL');
|
||||
|
||||
@ -178,7 +215,7 @@ sub run
|
||||
|
||||
# Push the WAL
|
||||
$oHostDbMaster->executeSimple(
|
||||
"${strCommand} --compress --archive-async --process-max=2 ${strXlogPath}/${strSourceFile}",
|
||||
"${strCommandPush} --compress --archive-async --process-max=2 ${strXlogPath}/${strSourceFile}",
|
||||
{oLogTest => $self->expect()});
|
||||
push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}." . COMPRESS_EXT);
|
||||
|
||||
@ -203,6 +240,7 @@ sub run
|
||||
# Test that the WAL was pushed
|
||||
$self->archiveCheck($strSourceFile, $strArchiveChecksum, true, $oHostDbMaster->spoolPath());
|
||||
|
||||
# Remove from spool
|
||||
storageTest()->remove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@ -213,7 +251,11 @@ sub run
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
$strCommandPush . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandGet . " ${strSourceFile1} ${strXlogPath}/RECOVERYXLOG",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@ -224,7 +266,11 @@ sub run
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
$strCommandPush . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandGet . " ${strSourceFile1} ${strXlogPath}/RECOVERYXLOG",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
|
||||
|
||||
# Restore the file to its original condition
|
||||
@ -236,15 +282,19 @@ sub run
|
||||
$oHostDbMaster->stop({strStanza => $oHostDbMaster->stanza()});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
{oLogTest => $self->expect(), iExpectedExitStatus => ERROR_STOP});
|
||||
$strCommandPush . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_STOP, oLogTest => $self->expect()});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandGet . " ${strSourceFile1} ${strXlogPath}/RECOVERYXLOG",
|
||||
{iExpectedExitStatus => ERROR_STOP, oLogTest => $self->expect()});
|
||||
|
||||
$oHostDbMaster->start({strStanza => $oHostDbMaster->stanza()});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
&log(INFO, ' WAL duplicate ok');
|
||||
|
||||
$oHostDbMaster->executeSimple($strCommand . " ${strXlogPath}/${strSourceFile}", {oLogTest => $self->expect()});
|
||||
$oHostDbMaster->executeSimple($strCommandPush . " ${strXlogPath}/${strSourceFile}", {oLogTest => $self->expect()});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
&log(INFO, ' WAL duplicate error');
|
||||
@ -252,15 +302,40 @@ sub run
|
||||
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, $strSourceFile);
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
$strCommandPush . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $self->expect()});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
&log(INFO, ' get second WAL');
|
||||
|
||||
# Remove WAL so it can be recovered
|
||||
storageTest()->remove("${strXlogPath}/${strSourceFile}", {bIgnoreMissing => false});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandGet . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . " ${strSourceFile} ${strXlogPath}/RECOVERYXLOG",
|
||||
{oLogTest => $self->expect()});
|
||||
|
||||
# Check that the destination file exists
|
||||
if (storageDb()->exists("${strXlogPath}/RECOVERYXLOG"))
|
||||
{
|
||||
my ($strActualChecksum) = storageDb()->hashSize("${strXlogPath}/RECOVERYXLOG");
|
||||
|
||||
if ($strActualChecksum ne $strArchiveChecksum)
|
||||
{
|
||||
confess "recovered file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
confess "archive file '${strXlogPath}/RECOVERYXLOG' is not in destination";
|
||||
}
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
&log(INFO, ' .partial WAL');
|
||||
|
||||
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 2, "${strSourceFile}.partial");
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
$strCommandPush . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
{oLogTest => $self->expect()});
|
||||
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum, false);
|
||||
|
||||
@ -270,7 +345,7 @@ sub run
|
||||
&log(INFO, ' .partial WAL duplicate');
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}.partial", {oLogTest => $self->expect()});
|
||||
$strCommandPush . " ${strXlogPath}/${strSourceFile}.partial", {oLogTest => $self->expect()});
|
||||
$self->archiveCheck(
|
||||
"${strSourceFile}.partial", $strArchiveChecksum, false);
|
||||
|
||||
@ -279,7 +354,7 @@ sub run
|
||||
|
||||
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, "${strSourceFile}.partial");
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
$strCommandPush . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $self->expect()});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user