You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-10-30 23:37:45 +02:00
Storage and IO layer refactor:
Refactor storage layer to allow for new repository filesystems using drivers. (Reviewed by Cynthia Shang.) Refactor IO layer to allow for new compression formats, checksum types, and other capabilities using filters. (Reviewed by Cynthia Shang.)
This commit is contained in:
David Steele
@@ -21,12 +21,14 @@ use pgBackRest::Common::Ini;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Base;
|
||||
use pgBackRest::Storage::Filter::Gzip;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
use pgBackRestTest::Common::FileTest;
|
||||
use pgBackRestTest::Common::RunTest;
|
||||
|
||||
####################################################################################################################################
|
||||
@@ -49,16 +51,16 @@ sub run
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}")) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
|
||||
# Create hosts and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress});
|
||||
|
||||
# Create the xlog path
|
||||
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
filePathCreate($strXlogPath, undef, false, true);
|
||||
storageDb()->pathCreate($strXlogPath, {bCreateParent => true});
|
||||
|
||||
# Create the test path for pg_control and copy pg_control for stanza-create
|
||||
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
|
||||
storageDb()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
|
||||
executeTest(
|
||||
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
|
||||
DB_FILE_PGCONTROL);
|
||||
@@ -78,7 +80,9 @@ sub run
|
||||
|
||||
if (defined($self->expect()))
|
||||
{
|
||||
$self->expect()->supplementalAdd($oFile->pathGet(PATH_BACKUP_ARCHIVE) . '/archive.info');
|
||||
$self->expect()->supplementalAdd(
|
||||
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), undef,
|
||||
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
|
||||
}
|
||||
|
||||
if ($bExists)
|
||||
@@ -107,20 +111,29 @@ sub run
|
||||
}
|
||||
|
||||
# Change the directory permissions to enable file creation
|
||||
executeTest('sudo chmod 770 ' . dirname($oFile->pathGet(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . "-1")));
|
||||
filePathCreate(
|
||||
dirname(
|
||||
$oFile->pathGet(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . "-1/${strSourceFile}")), '0770', true, true);
|
||||
forceStorageMode(storageRepo(), STORAGE_REPO_ARCHIVE, '770');
|
||||
|
||||
$oFile->copy(
|
||||
PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file $strArchiveTestFile
|
||||
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . # Destination file
|
||||
"-1/${strSourceFile}",
|
||||
false, # Source is not compressed
|
||||
$bCompress, # Destination compress based on test
|
||||
undef, undef, # Unused params
|
||||
'0660', # Mode
|
||||
true); # Create path if it does not exist
|
||||
storageRepo()->pathCreate(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16),
|
||||
{strMode => '0770', bIgnoreExists => true, bCreateParent => true});
|
||||
|
||||
storageTest()->copy(
|
||||
$strArchiveTestFile,
|
||||
storageRepo()->openWrite(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . "-1/${strSourceFile}",
|
||||
{rhyFilter => $bCompress ? [{strClass => STORAGE_FILTER_GZIP}] : undef,
|
||||
strMode => '0660', bCreateParent => true}));
|
||||
|
||||
my ($strActualChecksum) = storageRepo()->hashSize(
|
||||
storageRepo()->openRead(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . "-1/${strSourceFile}",
|
||||
{rhyFilter => $bCompress ?
|
||||
[{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef}));
|
||||
|
||||
if ($strActualChecksum ne $strArchiveChecksum)
|
||||
{
|
||||
confess "archive file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
|
||||
}
|
||||
|
||||
my $strDestinationFile = "${strXlogPath}/${strArchiveFile}";
|
||||
|
||||
@@ -130,16 +143,18 @@ sub run
|
||||
{oLogTest => $self->expect()});
|
||||
|
||||
# Check that the destination file exists
|
||||
if ($oFile->exists(PATH_DB_ABSOLUTE, $strDestinationFile))
|
||||
if (storageDb()->exists($strDestinationFile))
|
||||
{
|
||||
if ($oFile->hash(PATH_DB_ABSOLUTE, $strDestinationFile) ne $strArchiveChecksum)
|
||||
my ($strActualChecksum) = storageDb()->hashSize($strDestinationFile);
|
||||
|
||||
if ($strActualChecksum ne $strArchiveChecksum)
|
||||
{
|
||||
confess "archive file hash does not match ${strArchiveChecksum}";
|
||||
confess "recovered file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
confess 'archive file is not in destination';
|
||||
confess "archive file '${strDestinationFile}' is not in destination";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,9 +21,9 @@ use pgBackRest::Common::Ini;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
@@ -37,7 +37,6 @@ use pgBackRestTest::Common::RunTest;
|
||||
sub archiveCheck
|
||||
{
|
||||
my $self = shift;
|
||||
my $oFile = shift;
|
||||
my $strArchiveFile = shift;
|
||||
my $strArchiveChecksum = shift;
|
||||
my $bCompress = shift;
|
||||
@@ -56,18 +55,18 @@ sub archiveCheck
|
||||
|
||||
do
|
||||
{
|
||||
$bFound = $oFile->exists(PATH_BACKUP_ARCHIVE, $strArchiveCheck);
|
||||
$bFound = storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}");
|
||||
}
|
||||
while (!$bFound && waitMore($oWait));
|
||||
|
||||
if (!$bFound)
|
||||
{
|
||||
confess 'unable to find ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveCheck);
|
||||
confess 'unable to find ' . storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}");
|
||||
}
|
||||
|
||||
if (defined($strSpoolPath))
|
||||
{
|
||||
fileRemove("${strSpoolPath}/archive/" . $self->stanza() . "/out/${strArchiveFile}.ok");
|
||||
storageTest()->remove("${strSpoolPath}/archive/" . $self->stanza() . "/out/${strArchiveFile}.ok");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,20 +90,20 @@ sub run
|
||||
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, arc_async ${bArchiveAsync}", $self->processMax() == 1)) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => $bArchiveAsync});
|
||||
|
||||
# Create the xlog path
|
||||
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
filePathCreate($strXlogPath, undef, false, true);
|
||||
storageTest()->pathCreate($strXlogPath, {bCreateParent => true});
|
||||
|
||||
# Create the test path for pg_control
|
||||
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
|
||||
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
|
||||
|
||||
# Copy pg_control for stanza-create
|
||||
executeTest(
|
||||
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
|
||||
DB_FILE_PGCONTROL);
|
||||
storageTest()->copy(
|
||||
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin',
|
||||
$oHostDbMaster->dbBasePath() . qw{/} . DB_FILE_PGCONTROL);
|
||||
|
||||
my $strCommand =
|
||||
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() .
|
||||
@@ -113,12 +112,13 @@ sub run
|
||||
# Test missing archive.info file
|
||||
&log(INFO, ' test archive.info missing');
|
||||
my $strSourceFile1 = $self->walSegment(1, 1, 1);
|
||||
filePathCreate("${strXlogPath}/archive_status");
|
||||
my $strArchiveFile1 = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, $strSourceFile1);
|
||||
storageTest()->pathCreate("${strXlogPath}/archive_status");
|
||||
my $strArchiveFile1 = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, $strSourceFile1);
|
||||
|
||||
$oHostDbMaster->executeSimple($strCommand . " ${strXlogPath}/${strSourceFile1} --archive-max-mb=24",
|
||||
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile1}.error") if $bArchiveAsync;
|
||||
storageTest()->remove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile1}.error")
|
||||
if $bArchiveAsync;
|
||||
|
||||
# Create the archive info file
|
||||
$oHostBackup->stanzaCreate('create required data for stanza',
|
||||
@@ -145,13 +145,14 @@ sub run
|
||||
}
|
||||
|
||||
$strSourceFile = $self->walSegment(1, 1, $iArchiveNo);
|
||||
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 2, $strSourceFile);
|
||||
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 2, $strSourceFile);
|
||||
|
||||
&log(INFO, ' backup ' . sprintf('%02d', $iBackup) .
|
||||
', archive ' .sprintf('%02x', $iArchive) .
|
||||
" - ${strArchiveFile}");
|
||||
|
||||
my $strArchiveTmp = undef;
|
||||
# Create a temp file to make sure it is deleted later
|
||||
my $strArchiveTmp;
|
||||
|
||||
if ($iBackup == 1 && $iArchive == 2)
|
||||
{
|
||||
@@ -160,11 +161,11 @@ sub run
|
||||
|
||||
$strArchiveTmp =
|
||||
$oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
|
||||
substr($strSourceFile, 0, 16) . "/${strSourceFile}" .
|
||||
($bCompress ? ".$oFile->{strCompressExtension}" : '') . '.pgbackrest.tmp';
|
||||
substr($strSourceFile, 0, 16) . "/${strSourceFile}-${strArchiveChecksum}" . ($bCompress ? qw{.} .
|
||||
COMPRESS_EXT : '') . qw{.} . STORAGE_TEMP_EXT;
|
||||
|
||||
executeTest('sudo chmod 770 ' . dirname($strArchiveTmp));
|
||||
fileStringWrite($strArchiveTmp, 'JUNK');
|
||||
storageTest()->put($strArchiveTmp, 'JUNK');
|
||||
|
||||
if ($bRemote)
|
||||
{
|
||||
@@ -178,9 +179,9 @@ sub run
|
||||
{oLogTest => $self->expect()});
|
||||
push(
|
||||
@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}" .
|
||||
($bCompress ? ".$oFile->{strCompressExtension}" : ''));
|
||||
($bCompress ? qw{.} . COMPRESS_EXT : ''));
|
||||
|
||||
# Make sure the temp file no longer exists
|
||||
# Make sure the temp file no longer exists if it was created
|
||||
if (defined($strArchiveTmp))
|
||||
{
|
||||
my $oWait = waitInit(5);
|
||||
@@ -188,7 +189,7 @@ sub run
|
||||
|
||||
do
|
||||
{
|
||||
$bFound = fileExists($strArchiveTmp);
|
||||
$bFound = storageTest()->exists($strArchiveTmp);
|
||||
}
|
||||
while ($bFound && waitMore($oWait));
|
||||
|
||||
@@ -200,34 +201,40 @@ sub run
|
||||
|
||||
if ($iArchive == $iBackup)
|
||||
{
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok") if $bArchiveAsync;
|
||||
storageTest()->remove(
|
||||
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok")
|
||||
if $bArchiveAsync;
|
||||
|
||||
&log(INFO, ' test db version mismatch error');
|
||||
|
||||
$oHostBackup->infoMunge(
|
||||
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
|
||||
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
|
||||
storageTest()->remove(
|
||||
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error")
|
||||
if $bArchiveAsync;
|
||||
|
||||
&log(INFO, ' test db system-id mismatch error');
|
||||
|
||||
$oHostBackup->infoMunge(
|
||||
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}});
|
||||
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}});
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;;
|
||||
storageTest()->remove(
|
||||
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error")
|
||||
if $bArchiveAsync;
|
||||
|
||||
# Restore the file to its original condition
|
||||
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
|
||||
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
|
||||
|
||||
# Fail because the process was killed
|
||||
if ($iBackup == 1 && !$bCompress)
|
||||
@@ -264,70 +271,75 @@ sub run
|
||||
|
||||
$oHostDbMaster->executeSimple($strCommand . " ${strXlogPath}/${strSourceFile}", {oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok") if $bArchiveAsync;
|
||||
storageTest()->remove(
|
||||
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok")
|
||||
if $bArchiveAsync;
|
||||
|
||||
# Now it should break on archive duplication (because checksum is different
|
||||
&log(INFO, " test archive duplicate error");
|
||||
|
||||
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, $strSourceFile);
|
||||
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, $strSourceFile);
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
|
||||
storageTest()->remove(
|
||||
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error")
|
||||
if $bArchiveAsync;
|
||||
|
||||
# Test .partial archive
|
||||
&log(INFO, " test .partial archive");
|
||||
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 2, "${strSourceFile}.partial");
|
||||
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 2, "${strSourceFile}.partial");
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " --no-" . OPTION_REPO_SYNC . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
{oLogTest => $self->expect()});
|
||||
$self->archiveCheck(
|
||||
$oFile, "${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
|
||||
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
|
||||
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
|
||||
|
||||
push(
|
||||
@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}" .
|
||||
($bCompress ? ".$oFile->{strCompressExtension}" : ''));
|
||||
($bCompress ? qw{.} . COMPRESS_EXT : ''));
|
||||
|
||||
# Test .partial archive duplicate
|
||||
&log(INFO, ' test .partial archive duplicate');
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}.partial", {oLogTest => $self->expect()});
|
||||
$self->archiveCheck(
|
||||
$oFile, "${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
|
||||
"${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
|
||||
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
|
||||
|
||||
# Test .partial archive with different checksum
|
||||
&log(INFO, ' test .partial archive with different checksum');
|
||||
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, "${strSourceFile}.partial");
|
||||
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, "${strSourceFile}.partial");
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommand . " ${strXlogPath}/${strSourceFile}.partial",
|
||||
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $self->expect()});
|
||||
|
||||
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
|
||||
storageTest()->remove(
|
||||
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error")
|
||||
if $bArchiveAsync;
|
||||
}
|
||||
else
|
||||
{
|
||||
$self->archiveCheck(
|
||||
$oFile, $strSourceFile, $strArchiveChecksum, $bCompress,
|
||||
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
|
||||
$strSourceFile, $strArchiveChecksum, $bCompress, $bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oFile->list(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001')},
|
||||
sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
|
||||
'(' . join(', ', @stryExpectedWAL) . ')',
|
||||
'all WAL in archive', {iWaitSeconds => 5});
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
if (defined($self->expect()))
|
||||
{
|
||||
sleep(1); # Ugly hack to ensure repo is stable before checking files - replace in new tests
|
||||
$self->expect()->supplementalAdd($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
|
||||
$self->expect()->supplementalAdd(
|
||||
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), undef,
|
||||
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,14 +20,13 @@ use pgBackRest::Archive::ArchivePush;
|
||||
use pgBackRest::Archive::ArchivePushAsync;
|
||||
use pgBackRest::Archive::ArchivePushFile;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Lock ;
|
||||
use pgBackRest::Common::Lock;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::DbVersion;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Protocol::Common::Common;
|
||||
use pgBackRest::Protocol::Helper;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
@@ -48,21 +47,6 @@ sub initModule
|
||||
$self->{strRepoPath} = $self->testPath() . '/repo';
|
||||
$self->{strArchivePath} = "$self->{strRepoPath}/archive/" . $self->stanza();
|
||||
$self->{strSpoolPath} = "$self->{strArchivePath}/out";
|
||||
|
||||
# Create the local file object
|
||||
$self->{oFile} =
|
||||
new pgBackRest::File
|
||||
(
|
||||
$self->stanza(),
|
||||
$self->{strRepoPath},
|
||||
new pgBackRest::Protocol::Common::Common
|
||||
(
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
HOST_PROTOCOL_TIMEOUT # Protocol timeout
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
@@ -73,21 +57,23 @@ sub initTest
|
||||
my $self = shift;
|
||||
|
||||
# Create WAL path
|
||||
filePathCreate($self->{strWalStatusPath}, undef, true, true);
|
||||
storageTest()->pathCreate($self->{strWalStatusPath}, {bIgnoreExists => true, bCreateParent => true});
|
||||
|
||||
# Create archive info
|
||||
filePathCreate($self->{strArchivePath}, undef, true, true);
|
||||
storageTest()->pathCreate($self->{strArchivePath}, {bIgnoreExists => true, bCreateParent => true});
|
||||
|
||||
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false);
|
||||
my $oOption = $self->initOption();
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false, {bIgnoreMissing => true});
|
||||
$oArchiveInfo->create(PG_VERSION_94, WAL_VERSION_94_SYS_ID, true);
|
||||
|
||||
$self->{strArchiveId} = $oArchiveInfo->archiveId();
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
# initOption
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
sub initOption
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
@@ -103,6 +89,18 @@ sub run
|
||||
$self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 6);
|
||||
$self->optionSetTest($oOption, OPTION_ARCHIVE_TIMEOUT, 3);
|
||||
|
||||
return $oOption;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# run
|
||||
####################################################################################################################################
|
||||
sub run
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
my $oOption = $self->initOption();
|
||||
|
||||
################################################################################################################################
|
||||
if ($self->begin("ArchivePushFile::archivePushCheck"))
|
||||
{
|
||||
@@ -112,76 +110,74 @@ sub run
|
||||
my $strWalSegment = '000000010000000100000001';
|
||||
|
||||
$self->testResult(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
'(9.4-1, [undef], [undef])', "${strWalSegment} WAL not found");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalMajorPath = "$self->{strArchivePath}/9.4-1/" . substr($strWalSegment, 0, 16);
|
||||
my $strWalSegmentHash = "${strWalSegment}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
|
||||
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
|
||||
|
||||
filePathCreate($strWalMajorPath, undef, false, true);
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageTest()->pathCreate($strWalMajorPath, {bCreateParent => true});
|
||||
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testResult(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
'(9.4-1, 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27,' .
|
||||
" WAL segment ${strWalSegment} already exists in the archive with the same checksum\n" .
|
||||
'HINT: this is valid in some recovery scenarios but may also indicate a problem.)',
|
||||
"${strWalSegment} WAL found");
|
||||
|
||||
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageTest()->remove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegmentHash = "${strWalSegment}-10be15a0ab8e1653dfab18c83180e74f1507cab1";
|
||||
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testException(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strWalSegment} already exists in the archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegment = "${strWalSegment}.partial";
|
||||
$strWalSegmentHash = "${strWalSegment}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
|
||||
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
|
||||
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testResult(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
'(9.4-1, 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27,' .
|
||||
" WAL segment ${strWalSegment} already exists in the archive with the same checksum\n" .
|
||||
'HINT: this is valid in some recovery scenarios but may also indicate a problem.)',
|
||||
"${strWalSegment} WAL found");
|
||||
|
||||
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageTest()->remove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegmentHash = "${strWalSegment}-10be15a0ab8e1653dfab18c83180e74f1507cab1";
|
||||
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testException(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
|
||||
ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strWalSegment} already exists in the archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(sub {archivePushCheck(
|
||||
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID)},
|
||||
ERROR_ASSERT, "strFile is required in File->hash");
|
||||
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID)},
|
||||
ERROR_ASSERT, "xFileExp is required in Storage::Local->hashSize");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strHistoryFile = "00000001.history";
|
||||
|
||||
fileStringWrite("$self->{strArchivePath}/9.4-1/${strHistoryFile}");
|
||||
storageTest()->put("$self->{strArchivePath}/9.4-1/${strHistoryFile}");
|
||||
|
||||
$self->testResult(sub {archivePushCheck(
|
||||
$self->{oFile}, $strHistoryFile, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strHistoryFile}")},
|
||||
$strHistoryFile, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strHistoryFile}")},
|
||||
'(9.4-1, [undef], [undef])', "history file ${strHistoryFile} found");
|
||||
}
|
||||
|
||||
@@ -196,20 +192,18 @@ sub run
|
||||
$self->optionSetTest($oOption, OPTION_BACKUP_USER, $self->pgUser());
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
# Create the file object
|
||||
my $oRemoteFile = new pgBackRest::File(
|
||||
$self->stanza(), $self->{strRepoPath}, protocolGet(BACKUP, undef, {strBackRestBin => $self->backrestExe()}));
|
||||
protocolGet(BACKUP, undef, {strBackRestBin => $self->backrestExe()});
|
||||
|
||||
# Generate a normal segment
|
||||
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->testResult(
|
||||
sub {archivePushFile($oRemoteFile, $self->{strWalPath}, $strSegment, false, false)}, '[undef]',
|
||||
sub {archivePushFile($self->{strWalPath}, $strSegment, false, false)}, '[undef]',
|
||||
"${strSegment} WAL segment to remote");
|
||||
|
||||
$self->testResult(
|
||||
sub {archivePushFile($oRemoteFile, $self->{strWalPath}, $strSegment, false, false)},
|
||||
sub {archivePushFile($self->{strWalPath}, $strSegment, false, false)},
|
||||
"WAL segment 000000010000000100000001 already exists in the archive with the same checksum\n" .
|
||||
'HINT: this is valid in some recovery scenarios but may also indicate a problem.',
|
||||
"${strSegment} WAL duplicate segment to remote");
|
||||
@@ -226,6 +220,8 @@ sub run
|
||||
if ($self->begin("ArchivePush->readyList()"))
|
||||
{
|
||||
my $oPushAsync = new pgBackRest::Archive::ArchivePushAsync($self->{strWalPath}, $self->{strSpoolPath});
|
||||
$self->optionBoolSetTest($oOption, OPTION_ARCHIVE_ASYNC, true);
|
||||
$self->optionSetTest($oOption, OPTION_SPOOL_PATH, $self->{strRepoPath});
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
$oPushAsync->initServer();
|
||||
|
||||
@@ -234,35 +230,31 @@ sub run
|
||||
my $iWalMinor = 1;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.done');
|
||||
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.done');
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '()',
|
||||
'ignore files without .ready extension');
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '(000000010000000100000002, 000000010000000100000003)',
|
||||
'.ready files are found');
|
||||
|
||||
fileStringWrite("$self->{strSpoolPath}/000000010000000100000002.ok");
|
||||
fileStringWrite("$self->{strSpoolPath}/000000010000000100000003.ok");
|
||||
storageTest()->put("$self->{strSpoolPath}/000000010000000100000002.ok");
|
||||
storageTest()->put("$self->{strSpoolPath}/000000010000000100000003.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->walGenerate(
|
||||
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '(000000010000000100000004)',
|
||||
'new .ready files are found and duplicates ignored');
|
||||
|
||||
fileStringWrite("$self->{strSpoolPath}/000000010000000100000004.ok");
|
||||
storageTest()->put("$self->{strSpoolPath}/000000010000000100000004.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
@@ -273,32 +265,33 @@ sub run
|
||||
$iWalTimeline++;
|
||||
$iWalMinor = 1;
|
||||
|
||||
fileStringWrite("$self->{strWalStatusPath}/00000002.history.ready");
|
||||
storageTest()->put("$self->{strWalStatusPath}/00000002.history.ready");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '(00000002.history)',
|
||||
'history .ready file');
|
||||
|
||||
fileStringWrite("$self->{strSpoolPath}/00000002.history.ok");
|
||||
storageTest()->put("$self->{strSpoolPath}/00000002.history.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
fileStringWrite(
|
||||
storageTest()->put(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.00000028.backup.ready');
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '(000000020000000100000001.00000028.backup)',
|
||||
'backup .ready file');
|
||||
|
||||
fileStringWrite("$self->{strSpoolPath}/000000020000000100000001.00000028.backup.ok");
|
||||
storageTest()->put("$self->{strSpoolPath}/000000020000000100000001.00000028.backup.ok");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
fileRemove("$self->{strWalStatusPath}/00000002.history.ready");
|
||||
storageTest()->remove("$self->{strWalStatusPath}/00000002.history.ready");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->readyList()}, '()', 'remove 00000002.history.ok file');
|
||||
|
||||
$self->testResult(
|
||||
sub {fileExists("$self->{strWalStatusPath}/00000002.history.ready")}, false, '00000002.history.ok is removed');
|
||||
sub {storageTest()->exists("$self->{strWalStatusPath}/00000002.history.ready")}, false,
|
||||
'00000002.history.ok is removed');
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
@@ -313,12 +306,9 @@ sub run
|
||||
my $iWalMinor = 1;
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
fileStringWrite(
|
||||
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPushAsync->dropList($oPushAsync->readyList())}, '()',
|
||||
@@ -365,7 +355,7 @@ sub run
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a bogus warning ok (if content is present there must be two lines)
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
fileStringWrite("$self->{strSpoolPath}/${strSegment}.ok", "Test Warning");
|
||||
storageTest()->put("$self->{strSpoolPath}/${strSegment}.ok", "Test Warning");
|
||||
|
||||
# Check status
|
||||
$self->testException(
|
||||
@@ -393,7 +383,7 @@ sub run
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate an invalid error
|
||||
fileStringWrite("$self->{strSpoolPath}/${strSegment}.error");
|
||||
storageTest()->put("$self->{strSpoolPath}/${strSegment}.error");
|
||||
|
||||
# Check status (will error because there are now two status files)
|
||||
$self->testException(
|
||||
@@ -403,7 +393,7 @@ sub run
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Remove the ok file
|
||||
fileRemove("$self->{strSpoolPath}/${strSegment}.ok");
|
||||
storageTest()->remove("$self->{strSpoolPath}/${strSegment}.ok");
|
||||
|
||||
# Check status
|
||||
$self->testException(
|
||||
@@ -421,7 +411,7 @@ sub run
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Change the error file to an ok file
|
||||
fileMove("$self->{strSpoolPath}/${strSegment}.error", "$self->{strSpoolPath}/${strSegment}.ok");
|
||||
storageTest()->move("$self->{strSpoolPath}/${strSegment}.error", "$self->{strSpoolPath}/${strSegment}.ok");
|
||||
|
||||
# Check status
|
||||
$self->testResult(
|
||||
@@ -445,7 +435,11 @@ sub run
|
||||
{
|
||||
my $oPushAsync = new pgBackRest::Archive::ArchivePushAsync(
|
||||
$self->{strWalPath}, $self->{strSpoolPath}, $self->backrestExe());
|
||||
|
||||
$self->optionBoolSetTest($oOption, OPTION_ARCHIVE_ASYNC, true);
|
||||
$self->optionSetTest($oOption, OPTION_SPOOL_PATH, $self->{strRepoPath});
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$oPushAsync->initServer();
|
||||
|
||||
my $iWalTimeline = 1;
|
||||
@@ -455,26 +449,26 @@ sub run
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a normal segment
|
||||
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
# Generate an error (.ready file withough a corresponding WAL file)
|
||||
my $strSegmentError = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
fileStringWrite("$self->{strWalStatusPath}/$strSegmentError.ready");
|
||||
storageTest()->put("$self->{strWalStatusPath}/$strSegmentError.ready");
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(2, 0, 1, 1)', "process ${strSegment}, ${strSegmentError}");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileList($self->{strSpoolPath})}, "(${strSegment}.ok, ${strSegmentError}.error)",
|
||||
sub {storageSpool->list($self->{strSpoolPath})}, "(${strSegment}.ok, ${strSegmentError}.error)",
|
||||
"${strSegment} pushed, ${strSegmentError} errored");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileStringRead("$self->{strSpoolPath}/$strSegmentError.error")},
|
||||
ERROR_FILE_OPEN . "\nraised on local-1 host: unable to open $self->{strWalPath}/${strSegmentError}",
|
||||
sub {${storageSpool()->get("$self->{strSpoolPath}/$strSegmentError.error")}},
|
||||
ERROR_FILE_OPEN . "\nraised on 'local-1' host: unable to open $self->{strWalPath}/${strSegmentError}",
|
||||
"test ${strSegmentError}.error contents");
|
||||
|
||||
# Remove pushed WAL file
|
||||
@@ -482,16 +476,16 @@ sub run
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Fix errored WAL file by providing a valid segment
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegmentError);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegmentError);
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "process ${strSegment}, ${strSegmentError}");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegmentError)}, "${strSegmentError}-$self->{strWalHash}",
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegmentError)}, "${strSegmentError}-$self->{strWalHash}",
|
||||
"${strSegmentError} WAL in archive");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "${strSegmentError}.ok", "${strSegmentError} pushed");
|
||||
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "${strSegmentError}.ok", "${strSegmentError} pushed");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Remove previously errored WAL file
|
||||
@@ -500,7 +494,7 @@ sub run
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "remove ${strSegmentError}.ready");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "${strSegmentError} removed");
|
||||
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "${strSegmentError} removed");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Enable compression
|
||||
@@ -510,46 +504,46 @@ sub run
|
||||
# Create history file
|
||||
my $strHistoryFile = "00000001.history";
|
||||
|
||||
fileStringWrite("$self->{strWalPath}/${strHistoryFile}");
|
||||
fileStringWrite("$self->{strWalStatusPath}/$strHistoryFile.ready");
|
||||
storageTest()->put("$self->{strWalPath}/${strHistoryFile}");
|
||||
storageTest()->put("$self->{strWalStatusPath}/$strHistoryFile.ready");
|
||||
|
||||
# Create backup file
|
||||
my $strBackupFile = "${strSegment}.00000028.backup";
|
||||
|
||||
fileStringWrite("$self->{strWalPath}/${strBackupFile}");
|
||||
fileStringWrite("$self->{strWalStatusPath}/$strBackupFile.ready");
|
||||
storageTest()->put("$self->{strWalPath}/${strBackupFile}");
|
||||
storageTest()->put("$self->{strWalStatusPath}/$strBackupFile.ready");
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(2, 0, 2, 0)', "end processing ${strHistoryFile}, ${strBackupFile}");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileList($self->{strSpoolPath})}, "(${strHistoryFile}.ok, ${strBackupFile}.ok)",
|
||||
sub {storageSpool()->list($self->{strSpoolPath})}, "(${strHistoryFile}.ok, ${strBackupFile}.ok)",
|
||||
"${strHistoryFile}, ${strBackupFile} pushed");
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->{oFile}->exists(PATH_BACKUP_ARCHIVE, "$self->{strArchiveId}/${strHistoryFile}")}, true,
|
||||
sub {storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/$self->{strArchiveId}/${strHistoryFile}")}, true,
|
||||
"${strHistoryFile} in archive");
|
||||
|
||||
$self->testResult(
|
||||
sub {$self->{oFile}->exists(PATH_BACKUP_ARCHIVE, "$self->{strArchiveId}/${strBackupFile}")}, true,
|
||||
sub {storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/$self->{strArchiveId}/${strBackupFile}")}, true,
|
||||
"${strBackupFile} in archive");
|
||||
|
||||
# Remove history and backup files
|
||||
fileRemove("$self->{strWalPath}/${strHistoryFile}");
|
||||
fileRemove("$self->{strWalStatusPath}/$strHistoryFile.ready");
|
||||
fileRemove("$self->{strWalPath}/${strBackupFile}");
|
||||
fileRemove("$self->{strWalStatusPath}/$strBackupFile.ready");
|
||||
storageTest()->remove("$self->{strWalPath}/${strHistoryFile}");
|
||||
storageTest()->remove("$self->{strWalStatusPath}/$strHistoryFile.ready");
|
||||
storageTest()->remove("$self->{strWalPath}/${strBackupFile}");
|
||||
storageTest()->remove("$self->{strWalStatusPath}/$strBackupFile.ready");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Generate a normal segment
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "processing ${strSegment}.gz");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
# Remove the WAL and process so the .ok file is removed
|
||||
@@ -557,24 +551,24 @@ sub run
|
||||
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "remove ${strSegment}.ready");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "${strSegment}.ok removed");
|
||||
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "${strSegment}.ok removed");
|
||||
|
||||
# Generate the same WAL again
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "processed duplicate ${strSegment}.gz");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "${strSegment}.ok", "${strSegment} pushed");
|
||||
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "${strSegment}.ok", "${strSegment} pushed");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileStringRead("$self->{strSpoolPath}/${strSegment}.ok")},
|
||||
sub {${storageSpool()->get("$self->{strSpoolPath}/${strSegment}.ok")}},
|
||||
"0\nWAL segment ${strSegment} already exists in the archive with the same checksum\n" .
|
||||
'HINT: this is valid in some recovery scenarios but may also indicate a problem.',
|
||||
"${strSegment}.ok warning status");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
# Remove the WAL
|
||||
@@ -598,21 +592,21 @@ sub run
|
||||
|
||||
foreach my $strSegment (@strySegment)
|
||||
{
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
}
|
||||
|
||||
# Process and check results
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(3, 3, 1, 0)', "process and drop files");
|
||||
|
||||
$self->testResult(
|
||||
sub {fileList($self->{strSpoolPath})}, '(' . join('.ok, ', @strySegment) . '.ok)',
|
||||
sub {storageSpool()->list($self->{strSpoolPath})}, '(' . join('.ok, ', @strySegment) . '.ok)',
|
||||
join(', ', @strySegment) . " ok drop files written");
|
||||
|
||||
foreach my $strSegment (@strySegment)
|
||||
{
|
||||
$self->testResult(
|
||||
sub {fileStringRead("$self->{strSpoolPath}/${strSegment}.ok")},
|
||||
$strSegment eq $strySegment[0] ? '' :
|
||||
sub {${storageSpool()->get("$self->{strSpoolPath}/${strSegment}.ok")}},
|
||||
$strSegment eq $strySegment[0] ? undef :
|
||||
"0\ndropped WAL file ${strSegment} because archive queue exceeded " . optionGet(OPTION_ARCHIVE_QUEUE_MAX) .
|
||||
' bytes',
|
||||
"verify ${strSegment} status");
|
||||
@@ -626,7 +620,7 @@ sub run
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "final process to remove ok files");
|
||||
|
||||
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "ok files removed");
|
||||
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "ok files removed");
|
||||
}
|
||||
|
||||
################################################################################################################################
|
||||
@@ -634,6 +628,10 @@ sub run
|
||||
{
|
||||
my $oPush = new pgBackRest::Archive::ArchivePush($self->backrestExe());
|
||||
|
||||
$self->optionReset($oOption, OPTION_ARCHIVE_ASYNC);
|
||||
$self->optionReset($oOption, OPTION_SPOOL_PATH);
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
my $iWalTimeline = 1;
|
||||
my $iWalMajor = 1;
|
||||
my $iWalMinor = 1;
|
||||
@@ -656,12 +654,12 @@ sub run
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->testResult(sub {$oPush->process("pg_xlog/${strSegment}")}, 0, "${strSegment} WAL pushed (with relative path)");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
@@ -672,11 +670,11 @@ sub run
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL dropped");
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
# Set more realistic queue max and allow segment to push
|
||||
@@ -685,7 +683,7 @@ sub run
|
||||
|
||||
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL pushed");
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
@@ -702,8 +700,8 @@ sub run
|
||||
|
||||
# Write an error file and verify that it doesn't error the first time around
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
filePathCreate($self->{strSpoolPath}, undef, undef, true);
|
||||
fileStringWrite("$self->{strSpoolPath}/${strSegment}.error", ERROR_ARCHIVE_TIMEOUT . "\ntest error");
|
||||
storageTest()->pathCreate($self->{strSpoolPath}, {bCreateParent => true});
|
||||
storageTest()->put("$self->{strSpoolPath}/${strSegment}.error", ERROR_ARCHIVE_TIMEOUT . "\ntest error");
|
||||
|
||||
$self->testException(
|
||||
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_ARCHIVE_TIMEOUT,
|
||||
@@ -712,31 +710,31 @@ sub run
|
||||
$self->testResult($oPush->{bConfessOnError}, true, "went through error loop");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
"${strSegment} WAL not in archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
# Write an OK file so the async process is not actually started
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
fileStringWrite("$self->{strSpoolPath}/${strSegment}.ok");
|
||||
storageTest()->put("$self->{strSpoolPath}/${strSegment}.ok");
|
||||
|
||||
$self->testResult(
|
||||
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0,
|
||||
"${strSegment} WAL pushed async from synthetic ok file");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, '[undef]',
|
||||
"${strSegment} WAL not in archive");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL pushed async");
|
||||
exit if ($iProcessId != $PID);
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
|
||||
"${strSegment} WAL in archive");
|
||||
|
||||
$self->walRemove($self->{strWalPath}, $strSegment);
|
||||
@@ -754,7 +752,7 @@ sub run
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
|
||||
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
|
||||
|
||||
$self->optionSetTest($oOption, OPTION_BACKUP_HOST, BOGUS);
|
||||
$self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 60);
|
||||
@@ -762,8 +760,8 @@ sub run
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
$self->testException(
|
||||
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_HOST_CONNECT,
|
||||
'remote process terminated on ' . BOGUS . ' host.*');
|
||||
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_FILE_READ,
|
||||
"process '" . BOGUS . " remote' terminated.*");
|
||||
exit if ($iProcessId != $PID);
|
||||
|
||||
# Disable async archiving
|
||||
|
||||
@@ -21,9 +21,9 @@ use pgBackRest::Common::Ini;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Common::Wait;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Manifest;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
use pgBackRest::Storage::Helper;
|
||||
|
||||
use pgBackRestTest::Env::HostEnvTest;
|
||||
use pgBackRestTest::Common::ExecuteTest;
|
||||
@@ -48,21 +48,23 @@ sub run
|
||||
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version'))) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true});
|
||||
|
||||
my $oStorage = storageRepo();
|
||||
|
||||
# Create compression extension
|
||||
my $strCompressExt = $bCompress ? ".$oFile->{strCompressExtension}" : '';
|
||||
my $strCompressExt = $bCompress ? qw{.} . COMPRESS_EXT : '';
|
||||
|
||||
# Create the xlog path
|
||||
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
filePathCreate($strXlogPath, undef, false, true);
|
||||
$oStorage->pathCreate($strXlogPath, {bCreateParent => true});
|
||||
|
||||
# Create the test path for pg_control and copy pg_control for stanza-create
|
||||
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
|
||||
executeTest(
|
||||
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
|
||||
DB_FILE_PGCONTROL);
|
||||
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
|
||||
storageTest()->copy(
|
||||
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin',
|
||||
$oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
|
||||
|
||||
# Create the archive info file
|
||||
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . OPTION_ONLINE});
|
||||
@@ -74,16 +76,16 @@ sub run
|
||||
if ($iError == 0)
|
||||
{
|
||||
$oHostBackup->infoMunge(
|
||||
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
|
||||
$oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
|
||||
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
|
||||
}
|
||||
|
||||
# Push two more segments with errors to exceed archive-max-mb
|
||||
$oHostDbMaster->archivePush(
|
||||
$strXlogPath, $strArchiveTestFile, 2, $iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
|
||||
$strXlogPath, $strArchiveTestFile, 2, $iError ? ERROR_FILE_READ : ERROR_ARCHIVE_MISMATCH);
|
||||
|
||||
$oHostDbMaster->archivePush(
|
||||
$strXlogPath, $strArchiveTestFile, 3, $iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
|
||||
$strXlogPath, $strArchiveTestFile, 3, $iError ? ERROR_FILE_READ : ERROR_ARCHIVE_MISMATCH);
|
||||
|
||||
# Now this segment will get dropped
|
||||
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 4);
|
||||
@@ -91,13 +93,14 @@ sub run
|
||||
# Fix the database version
|
||||
if ($iError == 0)
|
||||
{
|
||||
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
|
||||
$oHostBackup->infoRestore($oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
|
||||
}
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {$oFile->list(
|
||||
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001', {strExpression => '^(?!000000010000000100000002).+'})},
|
||||
sub {$oStorage->list(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001',
|
||||
{strExpression => '^(?!000000010000000100000002).+'})},
|
||||
"000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt}",
|
||||
'segment 2-4 not pushed (2 is pushed sometimes when remote but ignore)', {iWaitSeconds => 5});
|
||||
|
||||
@@ -105,8 +108,9 @@ sub run
|
||||
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 5);
|
||||
|
||||
$self->testResult(
|
||||
sub {$oFile->list(
|
||||
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001', {strExpression => '^(?!000000010000000100000002).+'})},
|
||||
sub {$oStorage->list(
|
||||
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001',
|
||||
{strExpression => '^(?!000000010000000100000002).+'})},
|
||||
"(000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt}, " .
|
||||
"000000010000000100000005-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt})",
|
||||
'segment 5 is pushed', {iWaitSeconds => 5});
|
||||
|
||||
@@ -11,14 +11,13 @@ use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
use Storable qw(dclone);
|
||||
|
||||
use pgBackRest::Archive::ArchiveCommon;
|
||||
use pgBackRest::Common::Exception;
|
||||
use pgBackRest::Common::Log;
|
||||
use pgBackRest::Config::Config;
|
||||
use pgBackRest::File;
|
||||
use pgBackRest::FileCommon;
|
||||
use pgBackRest::Protocol::Storage::Helper;
|
||||
|
||||
use pgBackRestTest::Env::Host::HostBackupTest;
|
||||
|
||||
@@ -95,58 +94,54 @@ sub run
|
||||
################################################################################################################################
|
||||
if ($self->begin("${strModule}::walSegmentFind()"))
|
||||
{
|
||||
my $strArchiveId = '9.4-1';
|
||||
my $oFile = new pgBackRest::File(
|
||||
$self->stanza(),
|
||||
$self->testPath(),
|
||||
new pgBackRest::Protocol::Common::Common(
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
HOST_PROTOCOL_TIMEOUT # Protocol timeout
|
||||
));
|
||||
my $oOption = {};
|
||||
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
|
||||
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->testPath());
|
||||
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
|
||||
|
||||
my $strArchivePath = $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveId);;
|
||||
my $strArchiveId = '9.4-1';
|
||||
my $strArchivePath = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalSegment = '000000010000000100000001ZZ';
|
||||
|
||||
$self->testException(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, ERROR_ASSERT, "${strWalSegment} is not a WAL segment");
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, ERROR_ASSERT,
|
||||
"${strWalSegment} is not a WAL segment");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegment = '000000010000000100000001';
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, undef, "${strWalSegment} WAL not found");
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, undef, "${strWalSegment} WAL not found");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testException(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment, .1)}, ERROR_ARCHIVE_TIMEOUT,
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, .1)}, ERROR_ARCHIVE_TIMEOUT,
|
||||
"could not find WAL segment ${strWalSegment} after 0.1 second(s)");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalMajorPath = "${strArchivePath}/" . substr($strWalSegment, 0, 16);
|
||||
my $strWalSegmentHash = "${strWalSegment}-53aa5d59515aa7288ae02ba414c009aed1ca73ad";
|
||||
|
||||
filePathCreate($strWalMajorPath, undef, false, true);
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true});
|
||||
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, substr($strWalSegment, 8, 16))}, $strWalSegmentHash,
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, substr($strWalSegment, 8, 16))}, $strWalSegmentHash,
|
||||
"${strWalSegment} WAL found without timeline");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
my $strWalSegmentHash2 = "${strWalSegment}-a0b0d38b8aa263e25b8ff52a0a4ba85b6be97f9b.gz";
|
||||
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash2}");
|
||||
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash2}");
|
||||
|
||||
$self->testException(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, ERROR_ARCHIVE_DUPLICATE,
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, ERROR_ARCHIVE_DUPLICATE,
|
||||
"duplicates found in archive for WAL segment ${strWalSegment}: ${strWalSegmentHash}, ${strWalSegmentHash2}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
@@ -154,31 +149,31 @@ sub run
|
||||
my $strWalSegmentHash3 = "${strWalSegment3}-dcdd09246e1918e88c67cf44b35edc23b803d879";
|
||||
my $strWalMajorPath3 = "${strArchivePath}/" . substr($strWalSegment3, 0, 16);
|
||||
|
||||
filePathCreate($strWalMajorPath3, undef, false, true);
|
||||
fileStringWrite("${strWalMajorPath3}/${strWalSegmentHash3}");
|
||||
storageRepo()->pathCreate($strWalMajorPath3, {bCreateParent => true});
|
||||
storageRepo()->put("${strWalMajorPath3}/${strWalSegmentHash3}");
|
||||
|
||||
$self->testException(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, substr($strWalSegment, 8, 16))}, ERROR_ARCHIVE_DUPLICATE,
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, substr($strWalSegment, 8, 16))}, ERROR_ARCHIVE_DUPLICATE,
|
||||
"duplicates found in archive for WAL segment XXXXXXXX" . substr($strWalSegment, 8, 16) .
|
||||
": ${strWalSegmentHash}, ${strWalSegmentHash2}, ${strWalSegmentHash3}");
|
||||
|
||||
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
fileRemove("${strWalMajorPath3}/${strWalSegmentHash3}");
|
||||
storageRepo()->remove("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageRepo()->remove("${strWalMajorPath3}/${strWalSegmentHash3}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash2,
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash2,
|
||||
"${strWalSegment} WAL found with compressed extension");
|
||||
|
||||
fileRemove("${strWalMajorPath}/${strWalSegmentHash2}");
|
||||
storageRepo()->remove("${strWalMajorPath}/${strWalSegmentHash2}");
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
$strWalSegment = $strWalSegment . '.partial';
|
||||
$strWalSegmentHash = "${strWalSegment}-996195c807713ef9262170043e7222cb150aef70";
|
||||
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash}");
|
||||
|
||||
$self->testResult(
|
||||
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
|
||||
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user