db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Storage and IO layer refactor:

Browse Source 
Refactor storage layer to allow for new repository filesystems using drivers. (Reviewed by Cynthia Shang.)
Refactor IO layer to allow for new compression formats, checksum types, and other capabilities using filters. (Reviewed by Cynthia Shang.)
This commit is contained in:
David Steele
2017-06-09 17:51:41 -04:00
parent 7e982f05f5
commit de7fc37f88
183 changed files with 17880 additions and 14734 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/lib/pgBackRestTest/Common/CiTest.pm
+22 -19
View File
@@ -22,7 +22,6 @@ use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::FileCommon;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
@@ -45,12 +44,12 @@ sub new
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strBackRestBase},
$self->{oStorage},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strBackRestBase'},
{name => 'oStorage'},
);
# Return from function and log return values if any
@@ -116,7 +115,7 @@ sub process
my $strConfigNotFirstOS = '--no-package';
$strConfig .=
" - PGB_TEST_VM=\"${strVm}\" PGB_BUILD_PARAM=\"--db=none\" PGB_TEST_PARAM=\"--module=" .
" - PGB_TEST_VM=\"${strVm}\" PGB_BUILD_PARAM=\"--db=none\" PGB_TEST_PARAM=\"--vm-max=2 --module=" .
join(' --module=', @stryModule) . ($bFirst ? '' : " ${strConfigNotFirst}") . "\"\n";
$bFirst = false;
@@ -146,7 +145,7 @@ sub process
{
$strConfig .=
" - PGB_TEST_VM=\"${strVm}\" PGB_BUILD_PARAM=\"--db=none\"" .
" PGB_TEST_PARAM=\"--module=full --test=${strTest} ${strConfigNotFirst} ${strConfigNotFirstOS}\"\n";
" PGB_TEST_PARAM=\"--vm-max=2 --module=full --test=${strTest} ${strConfigNotFirst} ${strConfigNotFirstOS}\"\n";
}
}
@@ -160,26 +159,30 @@ sub process
$strConfig .=
"\n" .
"before_install:\n" .
" - sudo apt-get -qq update\n" .
" - sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl" .
" libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl\n" .
" - git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl\n" .
' - cd ~/libdevel-cover-perl && git checkout debian/' . LIB_COVER_VERSION . " && debuild -i -us -uc -b\n" .
' - sudo dpkg -i ~/' . LIB_COVER_PACKAGE . "\n" .
' - ' . LIB_COVER_EXE . " -v\n" .
" - sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl" .
" libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man" .
" devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl python-pip\n" .
" - |\n" .
" # Build Devel::Cover\n" .
" git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl\n" .
' cd ~/libdevel-cover-perl && git checkout debian/' . LIB_COVER_VERSION . " && debuild -i -us -uc -b\n" .
' sudo dpkg -i ~/' . LIB_COVER_PACKAGE . "\n" .
' ' . LIB_COVER_EXE . " -v\n" .
"\n" .
"install:\n" .
" - sudo adduser --ingroup=\${USER?} --disabled-password --gecos \"\" " . BACKREST_USER . "\n" .
" - umask 0022\n" .
" - cd ~ && pwd && whoami && umask && groups\n" .
" - mv \${TRAVIS_BUILD_DIR?} " . BACKREST_EXE . "\n" .
" - rm -rf \${TRAVIS_BUILD_DIR?}\n" .
" - |\n" .
" # User Configuration\n" .
" sudo adduser --ingroup=\${USER?} --disabled-password --gecos \"\" " . BACKREST_USER . "\n" .
" umask 0022\n" .
" cd ~ && pwd && whoami && umask && groups\n" .
" mv \${TRAVIS_BUILD_DIR?} " . BACKREST_EXE . "\n" .
" rm -rf \${TRAVIS_BUILD_DIR?}\n" .
" - " . BACKREST_EXE . "/test/test.pl --vm-build --vm=\${PGB_TEST_VM?} \${PGB_BUILD_PARAM?}\n" .
"\n" .
"script:\n" .
" - " . BACKREST_EXE . "/test/test.pl --vm-host=u14 --vm-max=2 --vm=\${PGB_TEST_VM?} \${PGB_TEST_PARAM?}\n";
" - " . BACKREST_EXE . "/test/test.pl --vm-host=u14 --vm=\${PGB_TEST_VM?} \${PGB_TEST_PARAM?}\n";
fileStringWrite("$self->{strBackRestBase}/.travis.yml", $strConfig, false);
$self->{oStorage}->put('.travis.yml', $strConfig);
# Return from function and log return values if any
return logDebugReturn($strOperation);
test/lib/pgBackRestTest/Common/ContainerTest.pm
+55 -33
View File
@@ -20,7 +20,6 @@ use Getopt::Long qw(GetOptions);
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::FileCommon;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
@@ -144,6 +143,7 @@ sub backrestConfigCreate
####################################################################################################################################
sub containerWrite
{
my $oStorageDocker = shift;
my $strTempPath = shift;
my $strOS = shift;
my $strTitle = shift;
@@ -162,7 +162,7 @@ sub containerWrite
}
elsif (!defined($bPre))
{
containerWrite($strTempPath, $strOS, $strTitle, $strImageParent, $strImage, $strScript, $bForce, true);
containerWrite($oStorageDocker, $strTempPath, $strOS, $strTitle, $strImageParent, $strImage, $strScript, $bForce, true);
if (defined($bPreOnly) && $bPreOnly)
{
@@ -179,7 +179,7 @@ sub containerWrite
$strScript;
# Write the image
fileStringWrite("${strTempPath}/${strImage}", trim($strScript) . "\n", false);
$oStorageDocker->put("${strTempPath}/${strImage}", trim($strScript) . "\n");
executeTest('docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') .
" -f ${strTempPath}/${strImage} -t ${strTag} ${strTempPath}",
{bSuppressStdErr => true});
@@ -256,7 +256,7 @@ sub sudoSetup
elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN)
{
$strScript .=
"\nRUN apt-get -y install sudo && \\\n" .
"\nRUN apt-get -y install sudo && \\" .
"\n echo '%${strGroup} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers";
}
else
@@ -295,28 +295,34 @@ sub perlInstall
{
my $strOS = shift;
my $oVm = vmGet();
my $strImage =
"# Install Perl packages\n";
if ($strOS eq VM_CO6)
if ($oVm->{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL)
{
$strImage .=
'RUN yum install -y perl perl-Time-HiRes perl-parent perl-JSON perl-Digest-SHA perl-DBD-Pg';
$strImage .= 'RUN yum install -y perl';
if ($strOS eq VM_CO6)
{
$strImage .= ' perl-Time-HiRes perl-parent perl-JSON';
}
else
{
$strImage .= ' perl-JSON-PP';
}
$strImage .= ' perl-Digest-SHA perl-DBD-Pg';
}
elsif ($strOS eq VM_CO7)
{
$strImage .=
'RUN yum install -y perl perl-JSON-PP perl-Digest-SHA perl-DBD-Pg';
}
elsif ($strOS eq VM_U12 || $strOS eq VM_U14)
{
$strImage .=
'RUN apt-get install -y libdbd-pg-perl libdbi-perl libnet-daemon-perl libplrpc-perl libhtml-parser-perl';
}
elsif ($strOS eq VM_U16 || $strOS eq VM_D8)
elsif ($oVm->{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN)
{
$strImage .=
'RUN apt-get install -y libdbd-pg-perl libdbi-perl libhtml-parser-perl';
if ($strOS eq VM_U12 || $strOS eq VM_U14)
{
$strImage .= ' libnet-daemon-perl libplrpc-perl';
}
}
else
{
@@ -331,13 +337,14 @@ sub perlInstall
####################################################################################################################################
sub containerBuild
{
my $oStorageDocker = shift;
my $strVm = shift;
my $bVmForce = shift;
my $strDbVersionBuild = shift;
# Create temp path
my $strTempPath = dirname(abs_path($0)) . '/.vagrant/docker';
filePathCreate($strTempPath, '0770', true, true);
my $strTempPath = $oStorageDocker->pathGet('test/.vagrant/docker');
$oStorageDocker->pathCreate($strTempPath, {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
# Remove old images on force
if ($bVmForce)
@@ -365,6 +372,8 @@ sub containerBuild
executeTest("ssh-keygen -f ${strTempPath}/id_rsa -t rsa -b 1024 -N ''", {bSuppressStdErr => true});
}
# VM Images
################################################################################################################################
my $oVm = vmGet();
foreach my $strOS (sort(keys(%$oVm)))
@@ -393,7 +402,7 @@ sub containerBuild
{
$strScript .=
"RUN apt-get update && \\\n" .
" apt-get -y install openssh-server wget\n";
" apt-get -y install openssh-server wget vim net-tools iputils-ping\n";
$strScript .=
"\n# Fix root tty\n" .
@@ -497,7 +506,7 @@ sub containerBuild
}
# Write the image
containerWrite($strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strScript, $bVmForce, false);
containerWrite($oStorageDocker, $strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strScript, $bVmForce, false);
# Base pre image
###########################################################################################################################
@@ -509,7 +518,9 @@ sub containerBuild
perlInstall($strOS) . "\n";
# Write the image
containerWrite($strTempPath, $strOS, 'Base (Pre-Installed)', $strImageParent, $strImage, $strScript, $bVmForce, false);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Base (Pre-Installed)', $strImageParent, $strImage, $strScript, $bVmForce,
false);
# Build image
###########################################################################################################################
@@ -536,7 +547,7 @@ sub containerBuild
}
# Write the image
containerWrite($strTempPath, $strOS, 'Build', $strImageParent, $strImage, $strScript, $bVmForce, false);
containerWrite($oStorageDocker, $strTempPath, $strOS, 'Build', $strImageParent, $strImage, $strScript, $bVmForce, false);
# Copy Devel::Cover to host so it can be installed in other containers
if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN)
@@ -612,8 +623,9 @@ sub containerBuild
}
# Write the image
containerWrite($strTempPath, $strOS, $strTitle, $strImageParent, $strImage, $strScript, $bVmForce,
$bDocBuildVersion ? undef : true, $bDocBuildVersion ? undef : true);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, $strTitle, $strImageParent, $strImage, $strScript, $bVmForce,
$bDocBuildVersion ? undef : true, $bDocBuildVersion ? undef : true);
# Db doc image
########################################################################################################################
@@ -629,7 +641,8 @@ sub containerBuild
$strScript .= "\n\n" . sudoSetup($strOS, TEST_GROUP);
# Write the image
containerWrite($strTempPath, $strOS, "${strTitle} Doc", $strImageParent, $strImage, $strScript, $bVmForce);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, "${strTitle} Doc", $strImageParent, $strImage, $strScript, $bVmForce);
}
# Db test image
@@ -644,7 +657,9 @@ sub containerBuild
$strScript .= coverSetup($strOS);
# Write the image
containerWrite($strTempPath, $strOS, "${strTitle} Test", $strImageParent, $strImage, $strScript, $bVmForce, true, true);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, "${strTitle} Test", $strImageParent, $strImage, $strScript, $bVmForce, true,
true);
}
# Db test image (for synthetic tests)
@@ -659,7 +674,9 @@ sub containerBuild
$strScript .= coverSetup($strOS);
# Write the image
containerWrite($strTempPath, $strOS, 'Db Synthetic Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Db Synthetic Test', $strImageParent, $strImage, $strScript, $bVmForce, true,
true);
# Loop test image
########################################################################################################################
@@ -689,7 +706,8 @@ sub containerBuild
$strScript .= coverSetup($strOS);
# Write the image
containerWrite($strTempPath, $strOS, 'Loop Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Loop Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true);
# Backup image
###########################################################################################################################
@@ -721,7 +739,9 @@ sub containerBuild
$strScript .= "\n\n" . sudoSetup($strOS, TEST_GROUP);
# Write the image
containerWrite($strTempPath, $strOS, "${strTitle} Doc", $strImageParent, $strImage, $strScript, $bVmForce, true, true);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, "${strTitle} Doc", $strImageParent, $strImage, $strScript, $bVmForce, true,
true);
}
# Backup Test image
@@ -745,7 +765,9 @@ sub containerBuild
$strScript .= coverSetup($strOS);
# Write the image
containerWrite($strTempPath, $strOS, "${strTitle} Test", $strImageParent, $strImage, $strScript, $bVmForce, true, true);
containerWrite(
$oStorageDocker, $strTempPath, $strOS, "${strTitle} Test", $strImageParent, $strImage, $strScript, $bVmForce, true,
true);
}
}
@@ -780,7 +802,7 @@ sub imageRemove
{
my $strRegExp = shift;
foreach my $strImage (sort(split("\n", trim(executeTest('docker images --format "{{.Repository}}"')))))
foreach my $strImage (sort(split("\n", trim(executeTest('docker images --format "{{.Repository}}/{{.Tag}}"')))))
{
if ($strImage =~ $strRegExp)
{
test/lib/pgBackRestTest/Common/DefineTest.pm
+117 -91
View File
@@ -58,11 +58,6 @@ use constant TESTDEF_COVERAGE_PARTIAL => false;
################################################################################################################################
# Code modules
################################################################################################################################
use constant TESTDEF_MODULE_FILE => 'File';
push @EXPORT, qw(TESTDEF_MODULE_FILE);
use constant TESTDEF_MODULE_FILE_COMMON => TESTDEF_MODULE_FILE . 'Common';
push @EXPORT, qw(TESTDEF_MODULE_FILE_COMMON);
use constant TESTDEF_MODULE_ARCHIVE => 'Archive';
push @EXPORT, qw(TESTDEF_MODULE_ARCHIVE);
use constant TESTDEF_MODULE_ARCHIVE_COMMON => TESTDEF_MODULE_ARCHIVE . '/ArchiveCommon';
@@ -108,7 +103,34 @@ my $oTestDef =
&TESTDEF_COVERAGE =>
{
&TESTDEF_MODULE_COMMON_INI => TESTDEF_COVERAGE_FULL,
&TESTDEF_MODULE_COMMON_INI => TESTDEF_COVERAGE_PARTIAL,
},
},
{
&TESTDEF_NAME => 'io-handle',
&TESTDEF_TOTAL => 7,
&TESTDEF_COVERAGE =>
{
'Common/Io/Handle' => TESTDEF_COVERAGE_FULL,
},
},
{
&TESTDEF_NAME => 'io-buffered',
&TESTDEF_TOTAL => 3,
&TESTDEF_COVERAGE =>
{
'Common/Io/Buffered' => TESTDEF_COVERAGE_FULL,
},
},
{
&TESTDEF_NAME => 'io-process',
&TESTDEF_TOTAL => 2,
&TESTDEF_COVERAGE =>
{
'Common/Io/Process' => TESTDEF_COVERAGE_PARTIAL,
},
},
]
@@ -148,75 +170,78 @@ my $oTestDef =
}
]
},
# File tests
# Storage tests
{
&TESTDEF_NAME => 'file',
&TESTDEF_NAME => 'storage',
&TESTDEF_CONTAINER => true,
&TESTDEF_COVERAGE =>
{
&TESTDEF_MODULE_FILE => TESTDEF_COVERAGE_PARTIAL,
&TESTDEF_MODULE_FILE_COMMON => TESTDEF_COVERAGE_PARTIAL,
},
&TESTDEF_TEST =>
[
{
&TESTDEF_NAME => 'unit',
&TESTDEF_TOTAL => 1,
&TESTDEF_NAME => 'filter-gzip',
&TESTDEF_TOTAL => 3,
&TESTDEF_COVERAGE =>
{
'Storage/Filter/Gzip' => TESTDEF_COVERAGE_PARTIAL,
},
},
{
&TESTDEF_NAME => 'owner',
&TESTDEF_TOTAL => 8,
},
{
&TESTDEF_NAME => 'path-create',
&TESTDEF_TOTAL => 8,
},
{
&TESTDEF_NAME => 'move',
&TESTDEF_TOTAL => 24,
},
{
&TESTDEF_NAME => 'compress',
&TESTDEF_TOTAL => 4,
},
{
&TESTDEF_NAME => 'wait',
&TESTDEF_NAME => 'filter-sha',
&TESTDEF_TOTAL => 2,
&TESTDEF_COVERAGE =>
{
'Storage/Filter/Sha' => TESTDEF_COVERAGE_FULL,
},
},
{
&TESTDEF_NAME => 'link',
&TESTDEF_NAME => 'posix',
&TESTDEF_TOTAL => 9,
&TESTDEF_COVERAGE =>
{
'Storage/Posix/Driver' => TESTDEF_COVERAGE_PARTIAL,
'Storage/Posix/FileRead' => TESTDEF_COVERAGE_PARTIAL,
'Storage/Posix/FileWrite' => TESTDEF_COVERAGE_PARTIAL,
},
},
{
&TESTDEF_NAME => 'local',
&TESTDEF_TOTAL => 9,
&TESTDEF_COVERAGE =>
{
'Storage/Local' => TESTDEF_COVERAGE_PARTIAL,
},
},
{
&TESTDEF_NAME => 'helper',
&TESTDEF_TOTAL => 4,
&TESTDEF_COVERAGE =>
{
'Storage/Helper' => TESTDEF_COVERAGE_PARTIAL,
},
},
]
},
# Protocol tests
{
&TESTDEF_NAME => 'protocol',
&TESTDEF_CONTAINER => true,
&TESTDEF_TEST =>
[
{
&TESTDEF_NAME => 'common-minion',
&TESTDEF_TOTAL => 1,
&TESTDEF_COVERAGE =>
{
'Protocol/Base/Minion' => TESTDEF_COVERAGE_PARTIAL,
},
},
{
&TESTDEF_NAME => 'stat',
&TESTDEF_TOTAL => 1,
},
{
&TESTDEF_NAME => 'manifest',
&TESTDEF_TOTAL => 5,
},
{
&TESTDEF_NAME => 'list',
&TESTDEF_TOTAL => 72,
},
{
&TESTDEF_NAME => 'remove',
&TESTDEF_TOTAL => 32,
},
{
&TESTDEF_NAME => 'hash',
&TESTDEF_TOTAL => 16,
},
{
&TESTDEF_NAME => 'exists',
&TESTDEF_TOTAL => 6,
},
{
&TESTDEF_NAME => 'copy',
&TESTDEF_TOTAL => 144,
}
]
},
# Info tests
@@ -237,34 +262,6 @@ my $oTestDef =
},
]
},
# Stanza tests
{
&TESTDEF_NAME => 'stanza',
&TESTDEF_COVERAGE =>
{
&TESTDEF_MODULE_STANZA => TESTDEF_COVERAGE_PARTIAL,
},
&TESTDEF_TEST =>
[
{
&TESTDEF_NAME => 'unit',
&TESTDEF_TOTAL => 2,
&TESTDEF_CONTAINER => true,
},
{
&TESTDEF_NAME => 'create',
&TESTDEF_TOTAL => 2,
&TESTDEF_EXPECT => true,
},
{
&TESTDEF_NAME => 'upgrade',
&TESTDEF_TOTAL => 2,
&TESTDEF_EXPECT => true,
},
]
},
# Archive tests
{
&TESTDEF_NAME => 'archive',
@@ -355,6 +352,36 @@ my $oTestDef =
},
]
},
# Stanza tests
{
&TESTDEF_NAME => 'stanza',
&TESTDEF_TEST =>
[
{
&TESTDEF_NAME => 'unit',
&TESTDEF_TOTAL => 7,
&TESTDEF_CONTAINER => true,
&TESTDEF_COVERAGE =>
{
&TESTDEF_MODULE_STANZA => TESTDEF_COVERAGE_FULL,
},
},
{
&TESTDEF_NAME => 'create',
&TESTDEF_TOTAL => 2,
&TESTDEF_EXPECT => true,
&TESTDEF_INDIVIDUAL => true,
},
{
&TESTDEF_NAME => 'upgrade',
&TESTDEF_TOTAL => 2,
&TESTDEF_EXPECT => true,
&TESTDEF_INDIVIDUAL => true,
},
]
},
# Full tests
{
&TESTDEF_NAME => 'full',
@@ -473,7 +500,6 @@ sub testDefModule
push @EXPORT, qw(testDefModule);
####################################################################################################################################
# testDefModuleTestList
####################################################################################################################################
test/lib/pgBackRestTest/Common/ExecuteTest.pm
+9 -14
View File
@@ -17,9 +17,10 @@ use IPC::Open3;
use POSIX ':sys_wait_h';
use Symbol 'gensym';
use pgBackRest::Common::Io::Handle;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Protocol::Common::Io::Process;
####################################################################################################################################
# new
@@ -99,7 +100,7 @@ sub begin
$self->{pId} = open3(undef, $self->{hOut}, $self->{hError}, $self->{strCommand});
# Create select objects
$self->{oIO} = new pgBackRest::Protocol::Common::Io::Process($self->{hOut}, undef, $self->{hError}, undef, undef, 30, 65536);
$self->{oIo} = new pgBackRest::Common::Io::Buffered(new pgBackRest::Common::Io::Handle('exec test', $self->{hOut}), 120, 65536);
if (!defined($self->{hError}))
{
@@ -135,16 +136,8 @@ sub endRetry
{
my $bFound = false;
# # Drain the stderr stream
# ??? This is a good idea but can only be done when the IO object has separate buffers for stdin and stderr
# while (my $strLine = $self->{oIO}->lineRead(0, false, false))
# {
# $bFound = true;
# $self->{strErrorLog} .= "$strLine\n";
# }
# Drain the stdout stream and look for test points
while (defined(my $strLine = $self->{oIO}->lineRead(0, true, false)))
while (defined(my $strLine = $self->{oIo}->readLine(true)))
{
$self->{strOutLog} .= "${strLine}\n";
$bFound = true;
@@ -176,7 +169,7 @@ sub endRetry
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
# Drain the stdout stream
while (defined(my $strLine = $self->{oIO}->lineRead(0, true, false)))
while (defined(my $strLine = $self->{oIo}->readLine(true)))
{
$self->{strOutLog} .= "${strLine}\n";
@@ -187,7 +180,9 @@ sub endRetry
}
# Drain the stderr stream
while (defined(my $strLine = $self->{oIO}->lineRead(0, false, false)))
my $oIoError = new pgBackRest::Common::Io::Buffered(new pgBackRest::Common::Io::Handle('exec test', $self->{hError}), 0, 65536);
while (defined(my $strLine = $oIoError->readLine(true, false)))
{
$self->{strErrorLog} .= "${strLine}\n";
}
@@ -236,7 +231,7 @@ sub endRetry
if ($self->{strErrorLog} ne '' && !$self->{bSuppressStdErr} && !$self->{bSuppressError})
{
confess &log(ERROR, "output found on STDERR:\n$self->{strErrorLog}");
confess &log(ERROR, "STDOUT:\n$self->{strOutLog}\n\noutput found on STDERR:\n$self->{strErrorLog}");
}
if ($self->{bShowOutput})
test/lib/pgBackRestTest/Common/FileTest.pm
+121 -46
View File
@@ -26,8 +26,8 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::Manifest;
use pgBackRest::Storage::Local;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::LogTest;
@@ -81,55 +81,10 @@ sub testPathRemove
my $bSuppressError = shift;
executeTest('sudo rm -rf ' . $strPath, {bSuppressError => $bSuppressError});
# remove_tree($strPath, {result => \my $oError});
#
# if (@$oError)
# {
# my $strMessage = "error(s) occurred while removing ${strPath}:";
#
# for my $strFile (@$oError)
# {
# $strMessage .= "\nunable to remove: " . $strFile;
# }
#
# confess $strMessage;
# }
}
push(@EXPORT, qw(testPathRemove));
####################################################################################################################################
# testPathCopy
#
# Copy a path.
####################################################################################################################################
sub testPathCopy
{
my $strSourcePath = shift;
my $strDestinationPath = shift;
my $bSuppressError = shift;
executeTest("cp -RpP ${strSourcePath} ${strDestinationPath}", {bSuppressError => $bSuppressError});
}
####################################################################################################################################
# testPathMove
#
# Copy a path.
####################################################################################################################################
sub testPathMove
{
my $strSourcePath = shift;
my $strDestinationPath = shift;
my $bSuppressError = shift;
testPathCopy($strSourcePath, $strDestinationPath, $bSuppressError);
testPathRemove($strSourcePath, $bSuppressError);
}
push(@EXPORT, qw(testPathMove));
####################################################################################################################################
# testFileCreate
#
@@ -182,4 +137,124 @@ sub testFileRemove
push(@EXPORT, qw(testFileRemove));
####################################################################################################################################
# forceStorageMode - force mode on a file or path
####################################################################################################################################
sub forceStorageMode
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strPathExp,
$strMode,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '::forceStorageMode', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'strMode'},
{name => 'bRecurse', optional => true, default => false},
);
executeTest('sudo chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp));
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageMode));
####################################################################################################################################
# forceStorageMove - force move a directory or file
####################################################################################################################################
sub forceStorageMove
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strSourcePathExp,
$strDestinationPathExp,
) =
logDebugParam
(
__PACKAGE__ . '->forceStorageMove', \@_,
{name => 'oStorage'},
{name => 'strSourcePathExp'},
{name => 'strDestinationPathExp'},
);
executeTest('sudo mv ' . $oStorage->pathGet($strSourcePathExp) . ' ' . $oStorage->pathGet($strDestinationPathExp));
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageMove));
####################################################################################################################################
# forceStorageOwner - force ownership on a file or path
####################################################################################################################################
sub forceStorageOwner
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strPathExp,
$strOwner,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '::forceStorageOwner', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'strOwner'},
{name => 'bRecurse', optional => true, default => false},
);
executeTest('sudo chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp));
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageOwner));
####################################################################################################################################
# forceStorageRemove - force remove a file or path from storage
####################################################################################################################################
sub forceStorageRemove
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strPathExp,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '->forceStorageRemove', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'bRecurse', optional => true, default => false},
);
executeTest('sudo rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp));
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageRemove));
1;
test/lib/pgBackRestTest/Common/HostGroupTest.pm
+7 -3
View File
@@ -57,12 +57,14 @@ sub hostAdd
my
(
$strOperation,
$oHost
$oHost,
$rstryHostName,
) =
logDebugParam
(
__PACKAGE__ . '->hostAdd', \@_,
{name => 'oHost'}
{name => 'oHost'},
{name => 'rstryHostName', optional => true},
);
$self->{host}{$oHost->{strName}} = $oHost;
@@ -70,6 +72,8 @@ sub hostAdd
$oHost->executeSimple("echo \"\" >> /etc/hosts", undef, 'root');
$oHost->executeSimple("echo \"# Test Hosts\" >> /etc/hosts", undef, 'root');
my $strHostList = $oHost->{strName} . (defined($rstryHostName) ? ' ' . join(' ', @{$rstryHostName}) : '');
# Iterate hosts to add IP mappings
foreach my $strOtherHostName (sort(keys(%{$self->{host}})))
{
@@ -78,7 +82,7 @@ sub hostAdd
if ($strOtherHostName ne $oHost->{strName})
{
# Add this host IP to all hosts
$oOtherHost->executeSimple("echo \"$oHost->{strIP} $oHost->{strName}\" >> /etc/hosts", undef, 'root');
$oOtherHost->executeSimple("echo \"$oHost->{strIP} ${strHostList}\" >> /etc/hosts", undef, 'root');
# Add all other host IPs to this host
$oHost->executeSimple("echo \"$oOtherHost->{strIP} ${strOtherHostName}\" >> /etc/hosts", undef, 'root');
test/lib/pgBackRestTest/Common/HostTest.pm
+5
View File
@@ -280,4 +280,9 @@ sub userGet
return $self->{strUser};
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub container {shift->{strContainer}}
1;
test/lib/pgBackRestTest/Common/JobTest.pm
+3 -2
View File
@@ -22,7 +22,6 @@ use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::FileCommon;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
@@ -44,6 +43,7 @@ sub new
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{oStorageTest},
$self->{strBackRestBase},
$self->{strTestPath},
$self->{strCoveragePath},
@@ -63,6 +63,7 @@ sub new
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oStorageTest'},
{name => 'strBackRestBase'},
{name => 'strTestPath'},
{name => 'strCoveragePath'},
@@ -139,7 +140,7 @@ sub run
if (!$self->{bDryRun} || $self->{bVmOut})
{
# Create host test directory
filePathCreate($strHostTestPath, '0770');
$self->{oStorageTest}->pathCreate($strHostTestPath, {strMode => '0770'});
if ($self->{oTest}->{&TEST_CONTAINER})
{
test/lib/pgBackRestTest/Common/LogTest.pm
+27 -10
View File
@@ -147,9 +147,7 @@ sub supplementalAdd
my $self = shift;
my $strFileName = shift;
my $strComment = shift;
open(my $hFile, '<', $strFileName)
or confess &log(ERROR, "unable to open ${strFileName} for appending to test log");
my $strContent = shift;
my $strHeader = "+ supplemental file: " . $self->regExpReplaceAll($strFileName);
@@ -160,12 +158,28 @@ sub supplementalAdd
$self->{strLog} .= "\n${strHeader}\n" . ('-' x length($strHeader)) . "\n";
while (my $strLine = readline($hFile))
if (!defined($strContent))
{
$self->{strLog} .= $self->regExpReplaceAll($strLine);
}
open(my $hFile, '<', $strFileName)
or confess &log(ERROR, "unable to open ${strFileName} for appending to test log");
close($hFile);
while (my $strLine = readline($hFile))
{
$self->{strLog} .= $self->regExpReplaceAll($strLine);
}
close($hFile);
}
else
{
if (defined($strContent) && length($strContent) > 0)
{
foreach my $strLine (split("\n", $strContent))
{
$self->{strLog} .= $self->regExpReplaceAll($strLine) . "\n";
}
}
}
}
####################################################################################################################################
@@ -356,12 +370,15 @@ sub regExpReplaceAll
$strLine = $self->regExpReplace($strLine, 'CONTAINER-EXEC', '^docker exec -u [a-z]*', '^docker exec -u [a-z]*', false);
$strLine = $self->regExpReplace($strLine, 'PROCESS-ID', 'sent term signal to process [0-9]+', '[0-9]+$', false);
$strLine = $self->regExpReplace($strLine, 'YEAR', ' \= backup\.history\/20[0-9]{2}', '20[0-9]{2}$');
$strLine = $self->regExpReplace($strLine, 'YEAR', 'backup\.history\/20[0-9]{2}', '20[0-9]{2}$');
$strLine = $self->regExpReplace($strLine, 'BACKUP-INCR', '[0-9]{8}\-[0-9]{6}F\_[0-9]{8}\-[0-9]{6}I');
$strLine = $self->regExpReplace($strLine, 'BACKUP-DIFF', '[0-9]{8}\-[0-9]{6}F\_[0-9]{8}\-[0-9]{6}D');
$strLine = $self->regExpReplace($strLine, 'BACKUP-FULL', '[0-9]{8}\-[0-9]{6}F');
$strLine = $self->regExpReplace(
$strLine, 'BACKUP-EXPR', 'strExpression \= \_[0-9]{8}\-[0-9]{6}', '\_[0-9]{8}\-[0-9]{6}$', false);
$strLine = $self->regExpReplace($strLine, 'GROUP', 'strGroup = [^ \n,\[\]]+', '[^ \n,\[\]]+$');
$strLine = $self->regExpReplace($strLine, 'GROUP', 'group"[ ]{0,1}:[ ]{0,1}"[^"]+', '[^"]+$');
$strLine = $self->regExpReplace($strLine, 'GROUP', 'group=\"[^"]+', '[^"]+$');
@@ -394,10 +411,10 @@ sub regExpReplaceAll
$strLine = $self->regExpReplace($strLine, 'VERSION',
"version[\"]{0,1}[ ]{0,1}[\:\=)]{1}[ ]{0,1}[\"]{0,1}" . BACKREST_VERSION, BACKREST_VERSION . '$');
$strLine = $self->regExpReplace($strLine, 'TIMESTAMP', 'timestamp"[ ]{0,1}:[ ]{0,1}[0-9]+','[0-9]{10}$');
$strLine = $self->regExpReplace($strLine, 'TIMESTAMP', 'timestamp"[ ]{0,1}:[ ]{0,1}[0-9]{10}','[0-9]{10}$');
$strLine = $self->regExpReplace($strLine, 'TIMESTAMP',
"timestamp-[a-z-]+[\"]{0,1}[ ]{0,1}[\:\=)]{1}[ ]{0,1}[\"]{0,1}[0-9]+", '[0-9]{10}$', false);
"timestamp-[a-z-]+[\"]{0,1}[ ]{0,1}[\:\=)]{1}[ ]{0,1}[\"]{0,1}[0-9]{10}", '[0-9]{10}$', false);
$strLine = $self->regExpReplace($strLine, 'TIMESTAMP', "start\" : [0-9]{10}", '[0-9]{10}$', false);
$strLine = $self->regExpReplace($strLine, 'TIMESTAMP', "stop\" : [0-9]{10}", '[0-9]{10}$', false);
$strLine = $self->regExpReplace($strLine, 'TIMESTAMP', TEST_GROUP . '\, [0-9]{10}', '[0-9]{10}$', false);
test/lib/pgBackRestTest/Common/RunTest.pm
+34 -7
View File
@@ -19,6 +19,9 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Storage::Posix::Driver;
use pgBackRest::Storage::Local;
use pgBackRestTest::Common::DefineTest;
use pgBackRestTest::Common::ExecuteTest;
@@ -36,6 +39,7 @@ use constant BOGUS => 'bogus';
# affecting the next run. Of course multiple subtests can be executed in a single run.
####################################################################################################################################
my $oTestRun;
my $oStorage;
####################################################################################################################################
# new
@@ -150,6 +154,9 @@ sub process
# Init will only be run on first test, clean/init on subsequent tests
$self->{bFirstTest} = true;
# Initialize test storage
$oStorage = new pgBackRest::Storage::Local($self->testPath(), new pgBackRest::Storage::Posix::Driver());
# Init, run, and end the test(s)
$self->initModule();
$self->run();
@@ -312,11 +319,13 @@ sub testResult
# Clear the cache for this test
logFileCacheClear();
my @stryResult;
do
{
eval
{
my @stryResult = ref($fnSub) eq 'CODE' ? $fnSub->() : $fnSub;
@stryResult = ref($fnSub) eq 'CODE' ? $fnSub->() : $fnSub;
if (@stryResult <= 1)
{
@@ -377,7 +386,11 @@ sub testResult
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
return logDebugReturn
(
$strOperation,
{name => 'result', value => \@stryResult, trace => true}
);
}
####################################################################################################################################
@@ -391,11 +404,12 @@ sub testException
my $strMessageExpected = shift;
# Output first line of the error message
my @stryErrorMessage = split('\n', $strMessageExpected);
&log(INFO, " [${iCodeExpected}] " . $stryErrorMessage[0]);
&log(INFO,
" [${iCodeExpected}] " . (defined($strMessageExpected) ? (split('\n', $strMessageExpected))[0] : 'undef error message'));
my $bError = false;
my $strError = "exception ${iCodeExpected}, \"${strMessageExpected}\" was expected";
my $strError =
"exception ${iCodeExpected}, " . (defined($strMessageExpected) ? "'${strMessageExpected}'" : '[UNDEF]') . " was expected";
eval
{
@@ -414,9 +428,14 @@ sub testException
}
if (!($EVAL_ERROR->code() == $iCodeExpected &&
($EVAL_ERROR->message() eq $strMessageExpected || $EVAL_ERROR->message() =~ $strMessageExpected)))
(!defined($strMessageExpected) && !defined($EVAL_ERROR->message()) ||
(defined($strMessageExpected) && defined($EVAL_ERROR->message()) &&
($EVAL_ERROR->message() eq $strMessageExpected || $EVAL_ERROR->message() =~ "^${strMessageExpected}" ||
$EVAL_ERROR->message() =~ "^${strMessageExpected} at ")))))
{
confess "${strError} but actual was " . $EVAL_ERROR->code() . ", \"" . $EVAL_ERROR->message() . "\"";
confess
"${strError} but actual was " . $EVAL_ERROR->code() . ', ' .
(defined($EVAL_ERROR->message()) ? qw{'} . $EVAL_ERROR->message() . qw{'} : '[UNDEF]');
}
$bError = true;
@@ -547,6 +566,14 @@ sub testRunExe
push(@EXPORT, qw(testRunExe));
####################################################################################################################################
# storageTest - get the storage for the current test
####################################################################################################################################
sub storageTest
{
return $oStorage;
}
push(@EXPORT, qw(storageTest));
####################################################################################################################################
# Getters
test/lib/pgBackRestTest/Common/VmTest.pm
+5 -4
View File
@@ -126,7 +126,7 @@ my $oyVm =
&VM_OS_BASE => VM_OS_BASE_RHEL,
&VM_OS => VM_OS_CENTOS,
&VM_IMAGE => 'centos:7',
&VM_CONTROL_MASTER => true,
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib64/perl5',
@@ -152,7 +152,7 @@ my $oyVm =
&VM_OS => VM_OS_DEBIAN,
&VM_OS_REPO => 'jessie',
&VM_IMAGE => 'debian:8',
&VM_CONTROL_MASTER => true,
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib/x86_64-linux-gnu/perl/5.20.2',
@@ -182,6 +182,7 @@ my $oyVm =
&VM_OS => VM_OS_UBUNTU,
&VM_OS_REPO => 'precise',
&VM_IMAGE => 'ubuntu:12.04',
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib/perl/5.14.2',
@@ -211,7 +212,7 @@ my $oyVm =
&VM_OS => VM_OS_UBUNTU,
&VM_OS_REPO => 'trusty',
&VM_IMAGE => 'ubuntu:14.04',
&VM_CONTROL_MASTER => true,
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib/perl/5.18.2',
@@ -244,7 +245,7 @@ my $oyVm =
&VM_OS => VM_OS_UBUNTU,
&VM_OS_REPO => 'xenial',
&VM_IMAGE => 'ubuntu:16.04',
&VM_CONTROL_MASTER => true,
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib/x86_64-linux-gnu/perl/5.22.1',
test/lib/pgBackRestTest/Env/ExpireEnvTest.pm
+51 -25
View File
@@ -11,6 +11,8 @@ use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(basename);
use pgBackRest::Archive::ArchiveInfo;
use pgBackRest::Backup::Common;
use pgBackRest::Backup::Info;
@@ -18,16 +20,17 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Stanza;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# new
@@ -45,7 +48,7 @@ sub new
my $strOperation,
$self->{oHostBackup},
$self->{strBackRestExe},
$self->{oFile},
$self->{oStorageRepo},
$self->{oLogTest},
$self->{oRunTest},
) =
@@ -54,7 +57,7 @@ sub new
__PACKAGE__ . '->new', \@_,
{name => 'oHostBackup', required => false, trace => true},
{name => 'strBackRestExe', trace => true},
{name => 'oFile', trace => true},
{name => 'oStorageRepo', trace => true},
{name => 'oLogTest', required => false, trace => true},
{name => 'oRunTest', required => false, trace => true},
);
@@ -107,8 +110,8 @@ sub stanzaSet
$$oStanza{iCatalogVersion} = $oStanzaCreate->{oDb}{iCatalogVersion};
$$oStanza{iControlVersion} = $oStanzaCreate->{oDb}{iControlVersion};
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{oFile}->pathGet(PATH_BACKUP_ARCHIVE));
my $oBackupInfo = new pgBackRest::Backup::Info($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER));
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE));
my $oBackupInfo = new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP));
if ($bStanzaUpgrade)
{
@@ -122,9 +125,9 @@ sub stanzaSet
}
# Get the archive and directory paths for the stanza
$$oStanza{strArchiveClusterPath} = $self->{oFile}->pathGet(PATH_BACKUP_ARCHIVE) . '/' . ($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oFile}->pathGet(PATH_BACKUP_CLUSTER);
filePathCreate($$oStanza{strArchiveClusterPath}, undef, undef, true);
$$oStanza{strArchiveClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE) . '/' . ($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP);
storageRepo()->pathCreate($$oStanza{strArchiveClusterPath}, {bCreateParent => true});
$self->{oStanzaHash}{$strStanza} = $oStanza;
@@ -159,7 +162,7 @@ sub stanzaCreate
my $strDbPath = optionGet(OPTION_DB_PATH);
# Create the test path for pg_control
filePathCreate(($strDbPath . '/' . DB_PATH_GLOBAL), undef, true);
storageTest()->pathCreate(($strDbPath . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true});
# Copy pg_control for stanza-create
executeTest(
@@ -199,7 +202,7 @@ sub stanzaUpgrade
$strDbVersionTemp =~ s/\.//;
# Remove pg_control
fileRemove(optionGet(OPTION_DB_PATH) . '/' . DB_FILE_PGCONTROL);
storageTest()->remove(optionGet(OPTION_DB_PATH) . '/' . DB_FILE_PGCONTROL);
# Copy pg_control for stanza-upgrade
executeTest(
@@ -257,7 +260,7 @@ sub backupCreate
$lTimestamp);
my $strBackupClusterSetPath = "$$oStanza{strBackupClusterPath}/${strBackupLabel}";
filePathCreate($strBackupClusterSetPath);
storageRepo()->pathCreate($strBackupClusterSetPath);
&log(INFO, "create backup ${strBackupLabel}");
@@ -296,9 +299,6 @@ sub backupCreate
$oManifest->save();
$$oStanza{oManifest} = $oManifest;
# Create the compressed history manifest
$self->{oFile}->compress(PATH_BACKUP_ABSOLUTE, $strManifestFile, false);
# Add the backup to info
my $oBackupInfo = new pgBackRest::Backup::Info($$oStanza{strBackupClusterPath}, false);
@@ -401,10 +401,10 @@ sub archiveCreate
do
{
my $strPath = "$$oStanza{strArchiveClusterPath}/" . substr($strArchive, 0, 16);
filePathCreate($strPath, undef, true);
storageRepo()->pathCreate($strPath, {bIgnoreExists => true});
my $strFile = "${strPath}/${strArchive}-0000000000000000000000000000000000000000" . ($iArchiveIdx % 2 == 0 ? '.gz' : '');
testFileCreate($strFile, 'ARCHIVE');
storageRepo()->put($strFile, 'ARCHIVE');
$iArchiveIdx++;
@@ -449,16 +449,42 @@ sub supplementalLog
if (defined($self->{oLogTest}))
{
$self->{oLogTest}->supplementalAdd($self->{oHostBackup}->repoPath() .
"/backup/${strStanza}/backup.info", $$oStanza{strBackupDescription});
$self->{oLogTest}->supplementalAdd(
$self->{oHostBackup}->repoPath() . "/backup/${strStanza}/backup.info", $$oStanza{strBackupDescription},
${storageRepo->get($self->{oHostBackup}->repoPath() . "/backup/${strStanza}/backup.info")});
executeTest(
'ls ' . $self->{oHostBackup}->repoPath() . "/backup/${strStanza} | grep -v \"backup.*\"",
{oLogTest => $self->{oLogTest}});
# Output backup list
$self->{oLogTest}->logAdd(
'ls ' . $self->{oHostBackup}->repoPath() . "/backup/${strStanza} | grep -v \"backup.*\"", undef,
join("\n", grep(!/^backup\.info.*$/i, storageRepo()->list("backup/${strStanza}"))));
executeTest(
'ls -R ' . $self->{oHostBackup}->repoPath() . "/archive/${strStanza} | grep -v \"archive.info\"",
{oLogTest => $self->{oLogTest}});
# Output archive manifest
my $rhManifest = storageRepo()->manifest(STORAGE_REPO_ARCHIVE);
my $strManifest;
my $strPrefix = '';
foreach my $strEntry (sort(keys(%{$rhManifest})))
{
# Skip files
next if $strEntry eq ARCHIVE_INFO_FILE || $strEntry eq ARCHIVE_INFO_FILE . INI_COPY_EXT;
if ($rhManifest->{$strEntry}->{type} eq 'd')
{
$strEntry = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE) . ($strEntry eq '.' ? '' : "/${strEntry}");
# &log(WARN, "DIR $strEntry");
$strManifest .= (defined($strManifest) ? "\n" : '') . "${strEntry}:\n";
$strPrefix = $strEntry;
}
else
{
# &log(WARN, "FILE $strEntry");
$strManifest .= basename($strEntry) . "\n";
}
}
$self->{oLogTest}->logAdd(
'ls -R ' . $self->{oHostBackup}->repoPath() . "/archive/${strStanza} | grep -v \"archive.info\"", undef, $strManifest);
}
return logDebugReturn($strOperation);
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
+106 -103
View File
@@ -16,14 +16,16 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::ArchiveInfo;
use pgBackRest::Backup::Common;
use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Posix::Driver;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBaseTest;
@@ -85,16 +87,19 @@ sub new
my $self = $class->SUPER::new($strName, {strImage => $strImage, strUser => $strUser});
bless $self, $class;
# Set parameters
# Create repo path
$self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO;
if ($$oParam{strBackupDestination} eq $self->nameGet())
{
$self->{strLogPath} = $self->repoPath() . '/' . HOST_PATH_LOG;
$self->{strLockPath} = $self->repoPath() . '/' . HOST_PATH_LOCK;
filePathCreate($self->repoPath(), '0770');
storageTest()->pathCreate($self->repoPath(), {strMode => '0770'});
}
# Set log/lock paths
$self->{strLogPath} = $self->testPath() . '/' . HOST_PATH_LOG;
$self->{strLockPath} = $self->testPath() . '/' . HOST_PATH_LOCK;
# Set conf file
$self->{strBackRestConfig} = $self->testPath() . '/' . BACKREST_CONF;
# Set LogTest object
@@ -109,18 +114,6 @@ sub new
# Default hardlink to false
$self->{bHardLink} = false;
# Create the local file object
$self->{oFile} = new pgBackRest::File(
$self->stanza(),
$self->repoPath(),
new pgBackRest::Protocol::Common::Common
(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
));
# Create a placeholder hash for file munging
$self->{hInfoFile} = {};
@@ -173,7 +166,7 @@ sub backupBegin
(defined($oExpectedManifest) ? " --no-online" : '') .
(defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') .
(defined($$oParam{bStandby}) && $$oParam{bStandby} ? " --backup-standby" : '') .
(defined($oParam->{bRepoLink}) && !$oParam->{bRepoLink} ? ' --no-repo-link' : '') .
(defined($oParam->{strRepoType}) ? " --repo-type=$oParam->{strRepoType}" : '') .
($strType ne 'incr' ? " --type=${strType}" : '') .
' --stanza=' . (defined($oParam->{strStanza}) ? $oParam->{strStanza} : $self->stanza()) . ' backup' .
(defined($strTest) ? " --test --test-delay=${fTestDelay} --test-point=" . lc($strTest) . '=y' : ''),
@@ -242,84 +235,75 @@ sub backupEnd
if (!$self->synthetic())
{
$oExpectedManifest = iniParse(
fileStringRead($self->repoPath() . '/backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST));
${storageRepo()->get(storageRepo()->pathGet('backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST))});
}
# Make sure tablespace links are correct
if ($strType eq BACKUP_TYPE_FULL || $self->hardLink())
if (($strType eq BACKUP_TYPE_FULL || $self->hardLink()) && $self->hasLink())
{
my $hTablespaceManifest = $self->{oFile}->manifest(
PATH_BACKUP_CLUSTER, "${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
my $hTablespaceManifest = storageRepo()->manifest(
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
# Remove . and ..
delete($hTablespaceManifest->{'.'});
delete($hTablespaceManifest->{'..'});
# Only check links if repo links are disabled
if (!defined($oParam->{bRepoLink}) || $oParam->{bRepoLink})
# Iterate file links
for my $strFile (sort(keys(%{$hTablespaceManifest})))
{
# Iterate file links
for my $strFile (sort(keys(%{$hTablespaceManifest})))
# Make sure the link is in the expected manifest
my $hManifestTarget =
$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"};
if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK ||
$hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile)
{
# Make sure the link is in the expected manifest
my $hManifestTarget =
$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"};
if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK ||
$hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile)
{
confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id");
}
# Make sure the link really is a link
if ($hTablespaceManifest->{$strFile}{type} ne 'l')
{
confess &log(ERROR, "'${strFile}' in tablespace directory is not a link");
}
# Make sure the link destination is correct
my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}";
if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination)
{
confess &log(ERROR,
"'${strFile}' link should reference '${strLinkDestination}' but actually references " .
"'$hTablespaceManifest->{$strFile}{link_destination}'");
}
confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id");
}
# Iterate manifest targets
for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}})))
# Make sure the link really is a link
if ($hTablespaceManifest->{$strFile}{type} ne 'l')
{
my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget};
my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID};
confess &log(ERROR, "'${strFile}' in tablespace directory is not a link");
}
# Make sure the target exists as a link on disk
if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) &&
!defined($hTablespaceManifest->{$strTablespaceId}))
{
confess &log(ERROR,
"target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'");
}
# Make sure the link destination is correct
my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}";
if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination)
{
confess &log(ERROR,
"'${strFile}' link should reference '${strLinkDestination}' but actually references " .
"'$hTablespaceManifest->{$strFile}{link_destination}'");
}
}
# Else make sure the tablespace directory is empty
elsif (keys(%{$hTablespaceManifest}) > 0)
# Iterate manifest targets
for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}})))
{
confess &log(ERROR, DB_PATH_PGTBLSPC . ' directory should be empty when --no-' . OPTION_REPO_LINK);
my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget};
my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID};
# Make sure the target exists as a link on disk
if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) &&
!defined($hTablespaceManifest->{$strTablespaceId}))
{
confess &log(ERROR,
"target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'");
}
}
}
# Else there should not be a tablespace directory at all
elsif ($self->{oFile}->exists(PATH_BACKUP_CLUSTER, "${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
elsif (storageRepo()->pathExists(STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
{
confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
}
# Check that latest link exists unless repo links are disabled
my $strLatestLink = $self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, LINK_LATEST);
my $bLatestLinkExists = fileExists($strLatestLink);
my $strLatestLink = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST);
my $bLatestLinkExists = storageRepo()->exists($strLatestLink);
if (!defined($oParam->{bRepoLink}) || $oParam->{bRepoLink})
if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq REPO_TYPE_POSIX) && $self->hasLink())
{
my $strLatestLinkDestination = readlink($strLatestLink);
@@ -330,7 +314,7 @@ sub backupEnd
}
elsif ($bLatestLinkExists)
{
confess &log(ERROR, "'" . LINK_LATEST . "' link should not exist when --no-" . OPTION_REPO_LINK);
confess &log(ERROR, "'" . LINK_LATEST . "' link should not exist");
}
# Only do compare for synthetic backups since for real backups the expected manifest *is* the actual manifest.
@@ -368,8 +352,12 @@ sub backupEnd
if ($self->synthetic() && $bManifestCompare)
{
$self->{oLogTest}->supplementalAdd($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST));
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), undef,
${storageRepo->get(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST)});
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
}
}
@@ -441,7 +429,7 @@ sub backupCompare
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup;
my $oActualManifest = new pgBackRest::Manifest(
$self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST));
storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST));
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} =
$oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START);
@@ -463,7 +451,7 @@ sub backupCompare
my $lRepoSize =
$oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
$oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
(fileStat($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/${strFileKey}.gz")))->size;
(storageTest->info(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/${strFileKey}.gz")))->size;
if (defined($lRepoSize) &&
$lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
@@ -495,13 +483,13 @@ sub backupCompare
my $strTestPath = $self->testPath();
fileStringWrite("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent}));
fileStringWrite("${strTestPath}/expected.manifest", iniRender($oExpectedManifest));
storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent}));
storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifest));
executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
fileRemove("${strTestPath}/expected.manifest");
fileRemove("${strTestPath}/actual.manifest");
storageTest()->remove("${strTestPath}/expected.manifest");
storageTest()->remove("${strTestPath}/actual.manifest");
# Return from function and log return values if any
return logDebugReturn($strOperation);
@@ -587,9 +575,8 @@ sub backupLast
{
my $self = shift;
my @stryBackup = $self->{oFile}->list(
PATH_BACKUP_CLUSTER, undef,
{strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'});
my @stryBackup = storageRepo()->list(
STORAGE_REPO_BACKUP, {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'});
if (!defined($stryBackup[0]))
{
@@ -755,15 +742,19 @@ sub stanzaCreate
# If the info file was created, then add it to the expect log
if (defined($self->{oLogTest}) && $self->synthetic() &&
fileExists($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info'))
storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO))
{
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
}
if (defined($self->{oLogTest}) && $self->synthetic() &&
fileExists($self->repoPath() . '/archive/' . $self->stanza() . '/archive.info'))
storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE))
{
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/archive/' . $self->stanza() . '/archive.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
}
# Return from function and log return values if any
@@ -808,15 +799,19 @@ sub stanzaUpgrade
# If the info file was created, then add it to the expect log
if (defined($self->{oLogTest}) && $self->synthetic() &&
fileExists($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info'))
storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO))
{
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
}
if (defined($self->{oLogTest}) && $self->synthetic() &&
fileExists($self->repoPath() . '/archive/' . $self->stanza() . '/archive.info'))
storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE))
{
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/archive/' . $self->stanza() . '/archive.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
}
# Return from function and log return values if any
@@ -924,10 +919,12 @@ sub configCreate
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_FILE} = lc(TRACE);
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_STDERR} = lc(OFF);
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_PATH} = $self->repoPath();
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_PATH} = $self->logPath();
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_LOCK_PATH} = $self->lockPath();
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_PROTOCOL_TIMEOUT} = 60;
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_DB_TIMEOUT} = 45;
if ($self->processMax() > 1)
{
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_PROCESS_MAX} = $self->processMax();
@@ -940,6 +937,8 @@ sub configCreate
if ($self->isHostBackup())
{
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_PATH} = $self->repoPath();
if (defined($$oParam{bHardlink}) && $$oParam{bHardlink})
{
$self->{bHardLink} = true;
@@ -1033,7 +1032,7 @@ sub configCreate
}
# Write out the configuration file
fileStringWrite($self->backrestConfig(), iniRender(\%oParamHash, true));
storageTest()->put($self->backrestConfig(), iniRender(\%oParamHash, true));
# Modify the file permissions so it can be read/saved by all test users
executeTest('sudo chmod 660 ' . $self->backrestConfig());
@@ -1065,7 +1064,7 @@ sub manifestMunge
{name => 'bCache', default => true},
);
$self->infoMunge($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST), $hParam, $bCache);
$self->infoMunge(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $hParam, $bCache);
# Return from function and log return values if any
return logDebugReturn($strOperation);
@@ -1092,7 +1091,7 @@ sub manifestRestore
{name => 'bSave', default => true},
);
$self->infoRestore($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST), $bSave);
$self->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $bSave);
# Return from function and log return values if any
return logDebugReturn($strOperation);
@@ -1128,12 +1127,13 @@ sub infoMunge
# If the original file content does not exist then load it
if (!defined($self->{hInfoFile}{$strFileName}))
{
$self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini($strFileName);
$self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini($strFileName, {oStorage => storageRepo()});
}
# Make a copy of the original file contents
my $oMungeIni = new pgBackRest::Common::Ini(
$strFileName, {bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent})});
$strFileName,
{bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), oStorage => storageRepo()});
# Load params
foreach my $strSection (keys(%{$hParam}))
@@ -1157,15 +1157,15 @@ sub infoMunge
}
# Modify the file/directory permissions so it can be saved
executeTest("sudo rm -f ${strFileName} && sudo chmod 770 " . dirname($strFileName));
executeTest("sudo rm -f ${strFileName}* && sudo chmod 770 " . dirname($strFileName));
# Save the munged data to the file
$oMungeIni->save();
# Fix permissions
executeTest(
"sudo chmod 640 ${strFileName} && sudo chmod 750 " . dirname($strFileName) .
' && sudo chown ' . $self->userGet() . " ${strFileName}");
executeTest(
"sudo chmod 640 ${strFileName}* && sudo chmod 750 " . dirname($strFileName) .
' && sudo chown ' . $self->userGet() . " ${strFileName}*");
# Clear the cache is requested
if (!$bCache)
@@ -1207,16 +1207,18 @@ sub infoRestore
if ($bSave)
{
# Modify the file/directory permissions so it can be saved
executeTest("sudo rm -f ${strFileName} && sudo chmod 770 " . dirname($strFileName));
executeTest("sudo rm -f ${strFileName}* && sudo chmod 770 " . dirname($strFileName));
# Save the munged data to the file
$self->{hInfoFile}{$strFileName}->{bModified} = true;
$self->{hInfoFile}{$strFileName}->save();
# Fix permissions
executeTest("sudo chmod 640 ${strFileName} && sudo chmod 750 " . dirname($strFileName));
executeTest(
"sudo chmod 640 ${strFileName} && sudo chmod 750 " . dirname($strFileName) .
' && sudo chown ' . $self->userGet() . " ${strFileName}*");
}
}
}
else
{
confess &log(ASSERT, "There is no original data cached for $strFileName. infoMunge must be called first.");
@@ -1236,6 +1238,7 @@ sub backrestConfig {return shift->{strBackRestConfig}}
sub backupDestination {return shift->{strBackupDestination}}
sub backrestExe {return testRunGet()->backrestExe()}
sub hardLink {return shift->{bHardLink}}
sub hasLink {storageRepo()->driver()->className() eq STORAGE_POSIX_DRIVER}
sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()}
sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER}
sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY}
test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm
+4 -2
View File
@@ -17,7 +17,7 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use pgBackRest::Common::Log;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
@@ -36,6 +36,8 @@ use constant HOST_DB_STANDBY => 'db-stand
push @EXPORT, qw(HOST_DB_STANDBY);
use constant HOST_BACKUP => 'backup';
push @EXPORT, qw(HOST_BACKUP);
use constant HOST_S3 => 's3-server';
push @EXPORT, qw(HOST_S3);
####################################################################################################################################
# new
@@ -59,7 +61,7 @@ sub new
);
my $strTestPath = testRunGet()->testPath() . ($strName eq HOST_BASE ? '' : "/${strName}");
filePathCreate($strTestPath, '0770');
storageTest()->pathCreate($strTestPath, {strMode => '0770'});
# Create the host
my $strProjectPath = dirname(dirname(abs_path($0)));
test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm
+36 -50
View File
@@ -25,15 +25,15 @@ use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# Host defaults
@@ -84,20 +84,10 @@ sub new
$self->{strDbBasePath} = $self->dbPath() . '/' . HOST_PATH_DB_BASE;
$self->{strTablespacePath} = $self->dbPath() . '/tablespace';
filePathCreate($self->dbBasePath(), undef, undef, true);
storageTest()->pathCreate($self->dbBasePath(), {strMode => '0700', bCreateParent => true});
if ($$oParam{strBackupDestination} ne $self->nameGet())
{
$self->{strSpoolPath} = $self->testPath() . '/' . HOST_PATH_SPOOL;
$self->{strLogPath} = $self->spoolPath() . '/' . HOST_PATH_LOG;
$self->{strLockPath} = $self->spoolPath() . '/' . HOST_PATH_LOCK;
filePathCreate($self->spoolPath());
}
else
{
$self->{strSpoolPath} = $self->repoPath();
}
$self->{strSpoolPath} = $self->testPath() . '/' . HOST_PATH_SPOOL;
storageTest()->pathCreate($self->spoolPath());
# Initialize linkRemap Hashes
$self->{hLinkRemap} = {};
@@ -143,16 +133,10 @@ sub archivePush
{
$strSourceFile = "${strXlogPath}/" . uc(sprintf('0000000100000001%08x', $iArchiveNo));
$self->{oFile}->copy(
PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false, # Destination is not compressed
undef, undef, undef, # Unused params
true); # Create path if it does not exist
storageTest()->copy($strArchiveTestFile, storageTest()->openWrite($strSourceFile, {bPathCreate => true}));
filePathCreate("${strXlogPath}/archive_status/", undef, true, true);
fileStringWrite("${strXlogPath}/archive_status/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)) . '.ready');
storageTest()->pathCreate("${strXlogPath}/archive_status/", {bIgnoreExists => true, bCreateParent => true});
storageTest()->put("${strXlogPath}/archive_status/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)) . '.ready');
}
$self->executeSimple(
@@ -160,7 +144,7 @@ sub archivePush
' --config=' . $self->backrestConfig() .
' --log-level-console=warn --archive-queue-max=' . int(2 * PG_WAL_SIZE) .
' --stanza=' . $self->stanza() .
(defined($iExpectedError) && $iExpectedError == ERROR_HOST_CONNECT ? ' --backup-host=bogus' : '') .
(defined($iExpectedError) && $iExpectedError == ERROR_FILE_READ ? ' --backup-host=bogus' : '') .
($bAsync ? '' : ' --no-archive-async') .
" archive-push" . (defined($strSourceFile) ? " ${strSourceFile}" : ''),
{iExpectedExitStatus => $iExpectedError, oLogTest => $self->{oLogTest}, bLogOutput => $self->synthetic()});
@@ -182,7 +166,7 @@ sub configRecovery
my $strStanza = $self->stanza();
# Load db config file
my $oConfig = iniParse(fileStringRead($self->backrestConfig()), {bRelaxed => true});
my $oConfig = iniParse(${storageTest->get($self->backrestConfig())}, {bRelaxed => true});
# Rewrite recovery options
my @stryRecoveryOption;
@@ -198,7 +182,7 @@ sub configRecovery
}
# Save db config file
fileStringWrite($self->backrestConfig(), iniRender($oConfig, true));
storageTest()->put($self->backrestConfig(), iniRender($oConfig, true));
}
####################################################################################################################################
@@ -214,7 +198,7 @@ sub configRemap
my $strStanza = $self->stanza();
# Load db config file
my $oConfig = iniParse(fileStringRead($self->backrestConfig()), {bRelaxed => true});
my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true});
# Load backup config file
my $oRemoteConfig;
@@ -224,7 +208,7 @@ sub configRemap
if (defined($oHostBackup))
{
$oRemoteConfig = iniParse(fileStringRead($oHostBackup->backrestConfig()), {bRelaxed => true});
$oRemoteConfig = iniParse(${storageTest()->get($oHostBackup->backrestConfig())}, {bRelaxed => true});
}
# Rewrite recovery section
@@ -262,7 +246,7 @@ sub configRemap
}
# Save db config file
fileStringWrite($self->backrestConfig(), iniRender($oConfig, true));
storageTest()->put($self->backrestConfig(), iniRender($oConfig, true));
# Save backup config file (but not is this is the standby which is not the source of backups)
if (defined($oHostBackup))
@@ -271,7 +255,7 @@ sub configRemap
executeTest(
'sudo chmod 660 ' . $oHostBackup->backrestConfig() . ' && sudo chmod 770 ' . dirname($oHostBackup->backrestConfig()));
fileStringWrite($oHostBackup->backrestConfig(), iniRender($oRemoteConfig, true));
storageTest()->put($oHostBackup->backrestConfig(), iniRender($oRemoteConfig, true));
# Fix permissions
executeTest(
@@ -369,9 +353,10 @@ sub restore
{
# Load the manifest
my $oExpectedManifest = new pgBackRest::Manifest(
$self->{oFile}->pathGet(
PATH_BACKUP_CLUSTER, ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST),
true);
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} . ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . qw{/} .
FILE_MANIFEST),
true);
$oExpectedManifestRef = $oExpectedManifest->{oContent};
@@ -481,17 +466,17 @@ sub restoreCompare
{
my $oExpectedManifest =
new pgBackRest::Manifest(
$self->{oFile}->pathGet(
PATH_BACKUP_CLUSTER,
($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) .
'/'. FILE_MANIFEST), true);
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} . ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) .
'/'. FILE_MANIFEST),
true);
$oLastManifest =
new pgBackRest::Manifest(
$self->{oFile}->pathGet(
PATH_BACKUP_CLUSTER,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} .
'/' . FILE_MANIFEST), true);
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} .
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST),
true);
}
# Generate the tablespace map for real backups
@@ -553,7 +538,7 @@ sub restoreCompare
$$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG});
$oActualManifest->build(
$self->{oFile}, ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}, $strDbClusterPath,
storageTest(), ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}, $strDbClusterPath,
$oLastManifest, false, $oTablespaceMap);
my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH);
@@ -614,13 +599,14 @@ sub restoreCompare
if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0)
{
my $oStat = fileStat($oActualManifest->dbPathGet($strSectionPath, $strName));
my $oStat = storageTest()->info($oActualManifest->dbPathGet($strSectionPath, $strName));
if ($oStat->blocks > 0 || S_ISLNK($oStat->mode))
{
my ($strHash) = storageTest()->hashSize($oActualManifest->dbPathGet($strSectionPath, $strName));
$oActualManifest->set(
MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM,
$self->{oFile}->hash(PATH_DB_ABSOLUTE, $oActualManifest->dbPathGet($strSectionPath, $strName)));
MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, $strHash);
}
else
{
@@ -723,13 +709,13 @@ sub restoreCompare
$self->manifestDefault($oExpectedManifestRef);
fileStringWrite("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent}));
fileStringWrite("${strTestPath}/expected.manifest", iniRender($oExpectedManifestRef));
storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent}));
storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifestRef));
executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
fileRemove("${strTestPath}/expected.manifest");
fileRemove("${strTestPath}/actual.manifest");
storageTest()->remove("${strTestPath}/expected.manifest");
storageTest()->remove("${strTestPath}/actual.manifest");
}
####################################################################################################################################
test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm
+11 -11
View File
@@ -23,8 +23,8 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::DbVersion;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
@@ -222,7 +222,7 @@ sub manifestFileCreate
my $strPathFile = $self->dbFileCreate($oManifestRef, $strTarget, $strFile, $strContent, $lTime, $strMode);
# Stat the file
my $oStat = fileStat($strPathFile);
my $oStat = storageTest()->info($strPathFile);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
@@ -336,7 +336,7 @@ sub manifestLinkCreate
my $strDbFile = $self->dbLinkCreate($oManifestRef, $strPath, $strFile, $strDestination);
# Stat the link
my $oStat = fileStat($strDbFile);
my $oStat = storageTest()->info($strDbFile);
# Check for errors in stat
if (!defined($oStat))
@@ -358,7 +358,7 @@ sub manifestLinkCreate
(defined(dirname($strPath)) ? dirname($strPath) : '') . "/${strDestination}";
}
$oStat = fileStat($strDestinationFile);
$oStat = storageTest()->info($strDestinationFile);
my $strSection = MANIFEST_SECTION_TARGET_PATH;
@@ -367,7 +367,7 @@ sub manifestLinkCreate
$strSection = MANIFEST_SECTION_TARGET_FILE;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM} = fileHash($strDestinationFile);
(${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM}) = storageTest()->hashSize($strDestinationFile);
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MASTER} =
defined($bMaster) ? ($bMaster ? JSON::PP::true : JSON::PP::false) : JSON::PP::false;
@@ -467,7 +467,7 @@ sub manifestPathCreate
my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strSubPath);
# Create the db path
my $strDbPath = $self->dbPathCreate($oManifestRef, $strPath, $strSubPath, $strMode);
my $strDbPath = $self->dbPathCreate($oManifestRef, $strPath, $strSubPath, defined($strMode) ? $strMode : '0700');
# Stat the file
my $oStat = lstat($strDbPath);
@@ -554,7 +554,7 @@ sub manifestTablespaceCreate
# Load linked path into manifest
my $strLinkPath = $self->tablespacePath($iOid);
my $strTarget = MANIFEST_TARGET_PGTBLSPC . "/${iOid}";
my $oStat = fileStat($strLinkPath);
my $oStat = storageTest()->info($strLinkPath);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
@@ -576,11 +576,11 @@ sub manifestTablespaceCreate
if (!-e $strTablespacePath)
{
filePathCreate($strTablespacePath, $strMode);
storageTest()->pathCreate($strTablespacePath, {strMode => defined($strMode) ? $strMode : '0700'});
}
# Load tablespace path into manifest
$oStat = fileStat($strTablespacePath);
$oStat = storageTest()->info($strTablespacePath);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGTBLSPC} =
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA};
@@ -597,7 +597,7 @@ sub manifestTablespaceCreate
or confess "unable to link ${strLink} to ${strLinkPath}";
# Load link into the manifest
$oStat = fileStat($strLink);
$oStat = storageTest()->info($strLink);
my $strLinkTarget = MANIFEST_TARGET_PGDATA . "/${strTarget}";
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
@@ -677,7 +677,7 @@ sub dbPathCreate
# Create the path
if (!(-e $strFinalPath))
{
filePathCreate($strFinalPath, $strMode);
storageTest()->pathCreate($strFinalPath, {strMode => $strMode});
}
return $strFinalPath;
test/lib/pgBackRestTest/Env/Host/HostDbTest.pm
+4 -4
View File
@@ -21,8 +21,8 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::DbVersion;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
@@ -123,7 +123,7 @@ sub new
}
# Create pg_xlog directory
filePathCreate($self->dbPath() . '/pg_xlog');
storageTest()->pathCreate($self->dbPath() . '/pg_xlog', {strMode => '0700'});
# Return from function and log return values if any
return logDebugReturn
@@ -476,13 +476,13 @@ sub clusterStop
}
# Grep for errors in postgresql.log
if (!$bIgnoreLogError && fileExists($self->pgLogFile()))
if (!$bIgnoreLogError && storageTest()->exists($self->pgLogFile()))
{
$self->executeSimple('grep ERROR ' . $self->pgLogFile(), {iExpectedExitStatus => 1});
}
# Remove the log file
fileRemove($self->pgLogFile(), true);
storageTest()->remove($self->pgLogFile(), {bIgnoreMissing => true});
}
####################################################################################################################################
test/lib/pgBackRestTest/Env/HostEnvTest.pm
+16 -41
View File
@@ -13,11 +13,11 @@ use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use Storable qw(dclone);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
@@ -25,6 +25,7 @@ use pgBackRestTest::Env::Host::HostDbCommonTest;
use pgBackRestTest::Env::Host::HostDbTest;
use pgBackRestTest::Env::Host::HostDbSyntheticTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# Constants
@@ -46,16 +47,6 @@ use constant WAL_VERSION_95 => '95';
use constant WAL_VERSION_95_SYS_ID => 6392579261579036436;
push @EXPORT, qw(WAL_VERSION_95_SYS_ID);
####################################################################################################################################
# initModule
####################################################################################################################################
sub initModule
{
# Set file and path modes
pathModeDefaultSet('0700');
fileModeDefaultSet('0600');
}
####################################################################################################################################
# setup
####################################################################################################################################
@@ -115,21 +106,6 @@ sub setup
$oHostGroup->hostAdd($oHostDbStandby);
}
# Create the local file object
my $oFile =
new pgBackRest::File
(
$oHostDbMaster->stanza(),
$oHostDbMaster->repoPath(),
new pgBackRest::Protocol::Common::Common
(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
)
);
# Create db master config
$oHostDbMaster->configCreate({
strBackupSource => $$oConfigParam{strBackupSource},
@@ -160,7 +136,14 @@ sub setup
bArchiveAsync => $$oConfigParam{bArchiveAsync}});
}
return $oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile;
# Set options needed for storage helper
my $oOption = {};
$self->optionSetTest($oOption, OPTION_DB_PATH, $oHostDbMaster->dbBasePath());
$self->optionSetTest($oOption, OPTION_REPO_PATH, $oHostBackup->repoPath());
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->testResult(sub {$self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH)}, '', 'config load');
return $oHostDbMaster, $oHostDbStandby, $oHostBackup;
}
####################################################################################################################################
@@ -171,7 +154,6 @@ sub setup
sub archiveGenerate
{
my $self = shift;
my $oFile = shift;
my $strXlogPath = shift;
my $iSourceNo = shift;
my $iArchiveNo = shift;
@@ -184,10 +166,7 @@ sub archiveGenerate
my $strSourceFile = "${strXlogPath}/${strArchiveFile}";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false); # Destination is not compressed
storageTest()->copy($strArchiveTestFile, $strSourceFile);
return $strArchiveFile, $strSourceFile;
}
@@ -215,7 +194,6 @@ sub walSegment
sub walGenerate
{
my $self = shift;
my $oFile = shift;
my $strWalPath = shift;
my $strPgVersion = shift;
my $iSourceNo = shift;
@@ -225,12 +203,9 @@ sub walGenerate
my $strWalFile = "${strWalPath}/${strWalSegment}" . (defined($bPartial) && $bPartial ? '.partial' : '');
my $strArchiveTestFile = $self->dataPath() . "/backup.wal${iSourceNo}_${strPgVersion}.bin";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strWalFile, # Destination file
false, # Source is not compressed
false); # Destination is not compressed
storageTest()->copy($strArchiveTestFile, $strWalFile);
fileStringWrite("${strWalPath}/archive_status/${strWalSegment}.ready");
storageTest()->put("${strWalPath}/archive_status/${strWalSegment}.ready");
return $strWalFile;
}
@@ -246,8 +221,8 @@ sub walRemove
my $strWalPath = shift;
my $strWalFile = shift;
fileRemove("$self->{strWalPath}/${strWalFile}");
fileRemove("$self->{strWalPath}/archive_status/${strWalFile}.ready");
storageTest()->remove("$self->{strWalPath}/${strWalFile}");
storageTest()->remove("$self->{strWalPath}/archive_status/${strWalFile}.ready");
}
1;
test/lib/pgBackRestTest/Module/Archive/ArchiveGetTest.pm
+39 -24
View File
@@ -21,12 +21,14 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Filter::Gzip;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
@@ -49,16 +51,16 @@ sub run
# Increment the run, log, and decide whether this unit test should be run
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}")) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
# Create hosts and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress});
# Create the xlog path
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
filePathCreate($strXlogPath, undef, false, true);
storageDb()->pathCreate($strXlogPath, {bCreateParent => true});
# Create the test path for pg_control and copy pg_control for stanza-create
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
storageDb()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
@@ -78,7 +80,9 @@ sub run
if (defined($self->expect()))
{
$self->expect()->supplementalAdd($oFile->pathGet(PATH_BACKUP_ARCHIVE) . '/archive.info');
$self->expect()->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
}
if ($bExists)
@@ -107,20 +111,29 @@ sub run
}
# Change the directory permissions to enable file creation
executeTest('sudo chmod 770 ' . dirname($oFile->pathGet(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . "-1")));
filePathCreate(
dirname(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . "-1/${strSourceFile}")), '0770', true, true);
forceStorageMode(storageRepo(), STORAGE_REPO_ARCHIVE, '770');
$oFile->copy(
PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file $strArchiveTestFile
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . # Destination file
"-1/${strSourceFile}",
false, # Source is not compressed
$bCompress, # Destination compress based on test
undef, undef, # Unused params
'0660', # Mode
true); # Create path if it does not exist
storageRepo()->pathCreate(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16),
{strMode => '0770', bIgnoreExists => true, bCreateParent => true});
storageTest()->copy(
$strArchiveTestFile,
storageRepo()->openWrite(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . "-1/${strSourceFile}",
{rhyFilter => $bCompress ? [{strClass => STORAGE_FILTER_GZIP}] : undef,
strMode => '0660', bCreateParent => true}));
my ($strActualChecksum) = storageRepo()->hashSize(
storageRepo()->openRead(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . "-1/${strSourceFile}",
{rhyFilter => $bCompress ?
[{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef}));
if ($strActualChecksum ne $strArchiveChecksum)
{
confess "archive file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
}
my $strDestinationFile = "${strXlogPath}/${strArchiveFile}";
@@ -130,16 +143,18 @@ sub run
{oLogTest => $self->expect()});
# Check that the destination file exists
if ($oFile->exists(PATH_DB_ABSOLUTE, $strDestinationFile))
if (storageDb()->exists($strDestinationFile))
{
if ($oFile->hash(PATH_DB_ABSOLUTE, $strDestinationFile) ne $strArchiveChecksum)
my ($strActualChecksum) = storageDb()->hashSize($strDestinationFile);
if ($strActualChecksum ne $strArchiveChecksum)
{
confess "archive file hash does not match ${strArchiveChecksum}";
confess "recovered file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
}
}
else
{
confess 'archive file is not in destination';
confess "archive file '${strDestinationFile}' is not in destination";
}
}
}
test/lib/pgBackRestTest/Module/Archive/ArchivePushTest.pm
+58 -46
View File
@@ -21,9 +21,9 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
@@ -37,7 +37,6 @@ use pgBackRestTest::Common::RunTest;
sub archiveCheck
{
my $self = shift;
my $oFile = shift;
my $strArchiveFile = shift;
my $strArchiveChecksum = shift;
my $bCompress = shift;
@@ -56,18 +55,18 @@ sub archiveCheck
do
{
$bFound = $oFile->exists(PATH_BACKUP_ARCHIVE, $strArchiveCheck);
$bFound = storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}");
}
while (!$bFound && waitMore($oWait));
if (!$bFound)
{
confess 'unable to find ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveCheck);
confess 'unable to find ' . storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}");
}
if (defined($strSpoolPath))
{
fileRemove("${strSpoolPath}/archive/" . $self->stanza() . "/out/${strArchiveFile}.ok");
storageTest()->remove("${strSpoolPath}/archive/" . $self->stanza() . "/out/${strArchiveFile}.ok");
}
}
@@ -91,20 +90,20 @@ sub run
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, arc_async ${bArchiveAsync}", $self->processMax() == 1)) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => $bArchiveAsync});
# Create the xlog path
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
filePathCreate($strXlogPath, undef, false, true);
storageTest()->pathCreate($strXlogPath, {bCreateParent => true});
# Create the test path for pg_control
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
# Copy pg_control for stanza-create
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
storageTest()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin',
$oHostDbMaster->dbBasePath() . qw{/} . DB_FILE_PGCONTROL);
my $strCommand =
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() .
@@ -113,12 +112,13 @@ sub run
# Test missing archive.info file
&log(INFO, ' test archive.info missing');
my $strSourceFile1 = $self->walSegment(1, 1, 1);
filePathCreate("${strXlogPath}/archive_status");
my $strArchiveFile1 = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, $strSourceFile1);
storageTest()->pathCreate("${strXlogPath}/archive_status");
my $strArchiveFile1 = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, $strSourceFile1);
$oHostDbMaster->executeSimple($strCommand . " ${strXlogPath}/${strSourceFile1} --archive-max-mb=24",
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile1}.error") if $bArchiveAsync;
storageTest()->remove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile1}.error")
if $bArchiveAsync;
# Create the archive info file
$oHostBackup->stanzaCreate('create required data for stanza',
@@ -145,13 +145,14 @@ sub run
}
$strSourceFile = $self->walSegment(1, 1, $iArchiveNo);
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 2, $strSourceFile);
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 2, $strSourceFile);
&log(INFO, ' backup ' . sprintf('%02d', $iBackup) .
', archive ' .sprintf('%02x', $iArchive) .
" - ${strArchiveFile}");
my $strArchiveTmp = undef;
# Create a temp file to make sure it is deleted later
my $strArchiveTmp;
if ($iBackup == 1 && $iArchive == 2)
{
@@ -160,11 +161,11 @@ sub run
$strArchiveTmp =
$oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
substr($strSourceFile, 0, 16) . "/${strSourceFile}" .
($bCompress ? ".$oFile->{strCompressExtension}" : '') . '.pgbackrest.tmp';
substr($strSourceFile, 0, 16) . "/${strSourceFile}-${strArchiveChecksum}" . ($bCompress ? qw{.} .
COMPRESS_EXT : '') . qw{.} . STORAGE_TEMP_EXT;
executeTest('sudo chmod 770 ' . dirname($strArchiveTmp));
fileStringWrite($strArchiveTmp, 'JUNK');
storageTest()->put($strArchiveTmp, 'JUNK');
if ($bRemote)
{
@@ -178,9 +179,9 @@ sub run
{oLogTest => $self->expect()});
push(
@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}" .
($bCompress ? ".$oFile->{strCompressExtension}" : ''));
($bCompress ? qw{.} . COMPRESS_EXT : ''));
# Make sure the temp file no longer exists
# Make sure the temp file no longer exists if it was created
if (defined($strArchiveTmp))
{
my $oWait = waitInit(5);
@@ -188,7 +189,7 @@ sub run
do
{
$bFound = fileExists($strArchiveTmp);
$bFound = storageTest()->exists($strArchiveTmp);
}
while ($bFound && waitMore($oWait));
@@ -200,34 +201,40 @@ sub run
if ($iArchive == $iBackup)
{
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok") if $bArchiveAsync;
storageTest()->remove(
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok")
if $bArchiveAsync;
&log(INFO, ' test db version mismatch error');
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
$oHostDbMaster->executeSimple(
$strCommand . " ${strXlogPath}/${strSourceFile}",
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
storageTest()->remove(
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error")
if $bArchiveAsync;
&log(INFO, ' test db system-id mismatch error');
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}});
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}});
$oHostDbMaster->executeSimple(
$strCommand . " ${strXlogPath}/${strSourceFile}",
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;;
storageTest()->remove(
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error")
if $bArchiveAsync;
# Restore the file to its original condition
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
# Fail because the process was killed
if ($iBackup == 1 && !$bCompress)
@@ -264,70 +271,75 @@ sub run
$oHostDbMaster->executeSimple($strCommand . " ${strXlogPath}/${strSourceFile}", {oLogTest => $self->expect()});
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok") if $bArchiveAsync;
storageTest()->remove(
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok")
if $bArchiveAsync;
# Now it should break on archive duplication (because checksum is different
&log(INFO, " test archive duplicate error");
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, $strSourceFile);
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, $strSourceFile);
$oHostDbMaster->executeSimple(
$strCommand . " ${strXlogPath}/${strSourceFile}",
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $self->expect()});
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
storageTest()->remove(
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error")
if $bArchiveAsync;
# Test .partial archive
&log(INFO, " test .partial archive");
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 2, "${strSourceFile}.partial");
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 2, "${strSourceFile}.partial");
$oHostDbMaster->executeSimple(
$strCommand . " --no-" . OPTION_REPO_SYNC . " ${strXlogPath}/${strSourceFile}.partial",
$strCommand . " ${strXlogPath}/${strSourceFile}.partial",
{oLogTest => $self->expect()});
$self->archiveCheck(
$oFile, "${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
push(
@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}" .
($bCompress ? ".$oFile->{strCompressExtension}" : ''));
($bCompress ? qw{.} . COMPRESS_EXT : ''));
# Test .partial archive duplicate
&log(INFO, ' test .partial archive duplicate');
$oHostDbMaster->executeSimple(
$strCommand . " ${strXlogPath}/${strSourceFile}.partial", {oLogTest => $self->expect()});
$self->archiveCheck(
$oFile, "${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
"${strSourceFile}.partial", $strArchiveChecksum, $bCompress,
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
# Test .partial archive with different checksum
&log(INFO, ' test .partial archive with different checksum');
$strArchiveFile = $self->walGenerate($oFile, $strXlogPath, WAL_VERSION_94, 1, "${strSourceFile}.partial");
$strArchiveFile = $self->walGenerate($strXlogPath, WAL_VERSION_94, 1, "${strSourceFile}.partial");
$oHostDbMaster->executeSimple(
$strCommand . " ${strXlogPath}/${strSourceFile}.partial",
{iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE, oLogTest => $self->expect()});
fileRemove($oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error") if $bArchiveAsync;
storageTest()->remove(
$oHostDbMaster->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.error")
if $bArchiveAsync;
}
else
{
$self->archiveCheck(
$oFile, $strSourceFile, $strArchiveChecksum, $bCompress,
$bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
$strSourceFile, $strArchiveChecksum, $bCompress, $bArchiveAsync ? $oHostDbMaster->spoolPath() : undef);
}
}
}
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oFile->list(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001')},
sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
'(' . join(', ', @stryExpectedWAL) . ')',
'all WAL in archive', {iWaitSeconds => 5});
#---------------------------------------------------------------------------------------------------------------------------
if (defined($self->expect()))
{
sleep(1); # Ugly hack to ensure repo is stable before checking files - replace in new tests
$self->expect()->supplementalAdd($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
$self->expect()->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
}
}
}
test/lib/pgBackRestTest/Module/Archive/ArchivePushUnitTest.pm
+123 -125
View File
@@ -20,14 +20,13 @@ use pgBackRest::Archive::ArchivePush;
use pgBackRest::Archive::ArchivePushAsync;
use pgBackRest::Archive::ArchivePushFile;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock ;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
@@ -48,21 +47,6 @@ sub initModule
$self->{strRepoPath} = $self->testPath() . '/repo';
$self->{strArchivePath} = "$self->{strRepoPath}/archive/" . $self->stanza();
$self->{strSpoolPath} = "$self->{strArchivePath}/out";
# Create the local file object
$self->{oFile} =
new pgBackRest::File
(
$self->stanza(),
$self->{strRepoPath},
new pgBackRest::Protocol::Common::Common
(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
)
);
}
####################################################################################################################################
@@ -73,21 +57,23 @@ sub initTest
my $self = shift;
# Create WAL path
filePathCreate($self->{strWalStatusPath}, undef, true, true);
storageTest()->pathCreate($self->{strWalStatusPath}, {bIgnoreExists => true, bCreateParent => true});
# Create archive info
filePathCreate($self->{strArchivePath}, undef, true, true);
storageTest()->pathCreate($self->{strArchivePath}, {bIgnoreExists => true, bCreateParent => true});
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false);
my $oOption = $self->initOption();
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false, {bIgnoreMissing => true});
$oArchiveInfo->create(PG_VERSION_94, WAL_VERSION_94_SYS_ID, true);
$self->{strArchiveId} = $oArchiveInfo->archiveId();
}
####################################################################################################################################
# run
# initOption
####################################################################################################################################
sub run
sub initOption
{
my $self = shift;
@@ -103,6 +89,18 @@ sub run
$self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 6);
$self->optionSetTest($oOption, OPTION_ARCHIVE_TIMEOUT, 3);
return $oOption;
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
my $oOption = $self->initOption();
################################################################################################################################
if ($self->begin("ArchivePushFile::archivePushCheck"))
{
@@ -112,76 +110,74 @@ sub run
my $strWalSegment = '000000010000000100000001';
$self->testResult(sub {archivePushCheck(
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
'(9.4-1, [undef], [undef])', "${strWalSegment} WAL not found");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalMajorPath = "$self->{strArchivePath}/9.4-1/" . substr($strWalSegment, 0, 16);
my $strWalSegmentHash = "${strWalSegment}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
$self->walGenerate(
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
filePathCreate($strWalMajorPath, undef, false, true);
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
storageTest()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(sub {archivePushCheck(
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
'(9.4-1, 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27,' .
" WAL segment ${strWalSegment} already exists in the archive with the same checksum\n" .
'HINT: this is valid in some recovery scenarios but may also indicate a problem.)',
"${strWalSegment} WAL found");
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
storageTest()->remove("${strWalMajorPath}/${strWalSegmentHash}");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegmentHash = "${strWalSegment}-10be15a0ab8e1653dfab18c83180e74f1507cab1";
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testException(sub {archivePushCheck(
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strWalSegment} already exists in the archive");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = "${strWalSegment}.partial";
$strWalSegmentHash = "${strWalSegment}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
$self->walGenerate(
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strWalSegment);
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(sub {archivePushCheck(
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
'(9.4-1, 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27,' .
" WAL segment ${strWalSegment} already exists in the archive with the same checksum\n" .
'HINT: this is valid in some recovery scenarios but may also indicate a problem.)',
"${strWalSegment} WAL found");
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
storageTest()->remove("${strWalMajorPath}/${strWalSegmentHash}");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegmentHash = "${strWalSegment}-10be15a0ab8e1653dfab18c83180e74f1507cab1";
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testException(sub {archivePushCheck(
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strWalSegment}")},
ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strWalSegment} already exists in the archive");
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {archivePushCheck(
$self->{oFile}, $strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID)},
ERROR_ASSERT, "strFile is required in File->hash");
$strWalSegment, PG_VERSION_94, WAL_VERSION_94_SYS_ID)},
ERROR_ASSERT, "xFileExp is required in Storage::Local->hashSize");
#---------------------------------------------------------------------------------------------------------------------------
my $strHistoryFile = "00000001.history";
fileStringWrite("$self->{strArchivePath}/9.4-1/${strHistoryFile}");
storageTest()->put("$self->{strArchivePath}/9.4-1/${strHistoryFile}");
$self->testResult(sub {archivePushCheck(
$self->{oFile}, $strHistoryFile, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strHistoryFile}")},
$strHistoryFile, PG_VERSION_94, WAL_VERSION_94_SYS_ID, "$self->{strWalPath}/${strHistoryFile}")},
'(9.4-1, [undef], [undef])', "history file ${strHistoryFile} found");
}
@@ -196,20 +192,18 @@ sub run
$self->optionSetTest($oOption, OPTION_BACKUP_USER, $self->pgUser());
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
# Create the file object
my $oRemoteFile = new pgBackRest::File(
$self->stanza(), $self->{strRepoPath}, protocolGet(BACKUP, undef, {strBackRestBin => $self->backrestExe()}));
protocolGet(BACKUP, undef, {strBackRestBin => $self->backrestExe()});
# Generate a normal segment
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->testResult(
sub {archivePushFile($oRemoteFile, $self->{strWalPath}, $strSegment, false, false)}, '[undef]',
sub {archivePushFile($self->{strWalPath}, $strSegment, false, false)}, '[undef]',
"${strSegment} WAL segment to remote");
$self->testResult(
sub {archivePushFile($oRemoteFile, $self->{strWalPath}, $strSegment, false, false)},
sub {archivePushFile($self->{strWalPath}, $strSegment, false, false)},
"WAL segment 000000010000000100000001 already exists in the archive with the same checksum\n" .
'HINT: this is valid in some recovery scenarios but may also indicate a problem.',
"${strSegment} WAL duplicate segment to remote");
@@ -226,6 +220,8 @@ sub run
if ($self->begin("ArchivePush->readyList()"))
{
my $oPushAsync = new pgBackRest::Archive::ArchivePushAsync($self->{strWalPath}, $self->{strSpoolPath});
$self->optionBoolSetTest($oOption, OPTION_ARCHIVE_ASYNC, true);
$self->optionSetTest($oOption, OPTION_SPOOL_PATH, $self->{strRepoPath});
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
$oPushAsync->initServer();
@@ -234,35 +230,31 @@ sub run
my $iWalMinor = 1;
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite(
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.done');
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.done');
$self->testResult(
sub {$oPushAsync->readyList()}, '()',
'ignore files without .ready extension');
#---------------------------------------------------------------------------------------------------------------------------
$self->walGenerate(
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->walGenerate(
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->testResult(
sub {$oPushAsync->readyList()}, '(000000010000000100000002, 000000010000000100000003)',
'.ready files are found');
fileStringWrite("$self->{strSpoolPath}/000000010000000100000002.ok");
fileStringWrite("$self->{strSpoolPath}/000000010000000100000003.ok");
storageTest()->put("$self->{strSpoolPath}/000000010000000100000002.ok");
storageTest()->put("$self->{strSpoolPath}/000000010000000100000003.ok");
#---------------------------------------------------------------------------------------------------------------------------
$self->walGenerate(
$self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->testResult(
sub {$oPushAsync->readyList()}, '(000000010000000100000004)',
'new .ready files are found and duplicates ignored');
fileStringWrite("$self->{strSpoolPath}/000000010000000100000004.ok");
storageTest()->put("$self->{strSpoolPath}/000000010000000100000004.ok");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
@@ -273,32 +265,33 @@ sub run
$iWalTimeline++;
$iWalMinor = 1;
fileStringWrite("$self->{strWalStatusPath}/00000002.history.ready");
storageTest()->put("$self->{strWalStatusPath}/00000002.history.ready");
$self->testResult(
sub {$oPushAsync->readyList()}, '(00000002.history)',
'history .ready file');
fileStringWrite("$self->{strSpoolPath}/00000002.history.ok");
storageTest()->put("$self->{strSpoolPath}/00000002.history.ok");
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite(
storageTest()->put(
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.00000028.backup.ready');
$self->testResult(
sub {$oPushAsync->readyList()}, '(000000020000000100000001.00000028.backup)',
'backup .ready file');
fileStringWrite("$self->{strSpoolPath}/000000020000000100000001.00000028.backup.ok");
storageTest()->put("$self->{strSpoolPath}/000000020000000100000001.00000028.backup.ok");
#---------------------------------------------------------------------------------------------------------------------------
fileRemove("$self->{strWalStatusPath}/00000002.history.ready");
storageTest()->remove("$self->{strWalStatusPath}/00000002.history.ready");
$self->testResult(
sub {$oPushAsync->readyList()}, '()', 'remove 00000002.history.ok file');
$self->testResult(
sub {fileExists("$self->{strWalStatusPath}/00000002.history.ready")}, false, '00000002.history.ok is removed');
sub {storageTest()->exists("$self->{strWalStatusPath}/00000002.history.ready")}, false,
'00000002.history.ok is removed');
}
################################################################################################################################
@@ -313,12 +306,9 @@ sub run
my $iWalMinor = 1;
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite(
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
fileStringWrite(
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
fileStringWrite(
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.ready');
$self->testResult(
sub {$oPushAsync->dropList($oPushAsync->readyList())}, '()',
@@ -365,7 +355,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
# Generate a bogus warning ok (if content is present there must be two lines)
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
fileStringWrite("$self->{strSpoolPath}/${strSegment}.ok", "Test Warning");
storageTest()->put("$self->{strSpoolPath}/${strSegment}.ok", "Test Warning");
# Check status
$self->testException(
@@ -393,7 +383,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
# Generate an invalid error
fileStringWrite("$self->{strSpoolPath}/${strSegment}.error");
storageTest()->put("$self->{strSpoolPath}/${strSegment}.error");
# Check status (will error because there are now two status files)
$self->testException(
@@ -403,7 +393,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
# Remove the ok file
fileRemove("$self->{strSpoolPath}/${strSegment}.ok");
storageTest()->remove("$self->{strSpoolPath}/${strSegment}.ok");
# Check status
$self->testException(
@@ -421,7 +411,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
# Change the error file to an ok file
fileMove("$self->{strSpoolPath}/${strSegment}.error", "$self->{strSpoolPath}/${strSegment}.ok");
storageTest()->move("$self->{strSpoolPath}/${strSegment}.error", "$self->{strSpoolPath}/${strSegment}.ok");
# Check status
$self->testResult(
@@ -445,7 +435,11 @@ sub run
{
my $oPushAsync = new pgBackRest::Archive::ArchivePushAsync(
$self->{strWalPath}, $self->{strSpoolPath}, $self->backrestExe());
$self->optionBoolSetTest($oOption, OPTION_ARCHIVE_ASYNC, true);
$self->optionSetTest($oOption, OPTION_SPOOL_PATH, $self->{strRepoPath});
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
$oPushAsync->initServer();
my $iWalTimeline = 1;
@@ -455,26 +449,26 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
# Generate a normal segment
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
# Generate an error (.ready file withough a corresponding WAL file)
my $strSegmentError = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
fileStringWrite("$self->{strWalStatusPath}/$strSegmentError.ready");
storageTest()->put("$self->{strWalStatusPath}/$strSegmentError.ready");
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(2, 0, 1, 1)', "process ${strSegment}, ${strSegmentError}");
$self->testResult(
sub {fileList($self->{strSpoolPath})}, "(${strSegment}.ok, ${strSegmentError}.error)",
sub {storageSpool->list($self->{strSpoolPath})}, "(${strSegment}.ok, ${strSegmentError}.error)",
"${strSegment} pushed, ${strSegmentError} errored");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
"${strSegment} WAL in archive");
$self->testResult(
sub {fileStringRead("$self->{strSpoolPath}/$strSegmentError.error")},
ERROR_FILE_OPEN . "\nraised on local-1 host: unable to open $self->{strWalPath}/${strSegmentError}",
sub {${storageSpool()->get("$self->{strSpoolPath}/$strSegmentError.error")}},
ERROR_FILE_OPEN . "\nraised on 'local-1' host: unable to open $self->{strWalPath}/${strSegmentError}",
"test ${strSegmentError}.error contents");
# Remove pushed WAL file
@@ -482,16 +476,16 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
# Fix errored WAL file by providing a valid segment
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegmentError);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegmentError);
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "process ${strSegment}, ${strSegmentError}");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegmentError)}, "${strSegmentError}-$self->{strWalHash}",
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegmentError)}, "${strSegmentError}-$self->{strWalHash}",
"${strSegmentError} WAL in archive");
$self->testResult(sub {fileList($self->{strSpoolPath})}, "${strSegmentError}.ok", "${strSegmentError} pushed");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "${strSegmentError}.ok", "${strSegmentError} pushed");
#---------------------------------------------------------------------------------------------------------------------------
# Remove previously errored WAL file
@@ -500,7 +494,7 @@ sub run
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "remove ${strSegmentError}.ready");
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "${strSegmentError} removed");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "${strSegmentError} removed");
#---------------------------------------------------------------------------------------------------------------------------
# Enable compression
@@ -510,46 +504,46 @@ sub run
# Create history file
my $strHistoryFile = "00000001.history";
fileStringWrite("$self->{strWalPath}/${strHistoryFile}");
fileStringWrite("$self->{strWalStatusPath}/$strHistoryFile.ready");
storageTest()->put("$self->{strWalPath}/${strHistoryFile}");
storageTest()->put("$self->{strWalStatusPath}/$strHistoryFile.ready");
# Create backup file
my $strBackupFile = "${strSegment}.00000028.backup";
fileStringWrite("$self->{strWalPath}/${strBackupFile}");
fileStringWrite("$self->{strWalStatusPath}/$strBackupFile.ready");
storageTest()->put("$self->{strWalPath}/${strBackupFile}");
storageTest()->put("$self->{strWalStatusPath}/$strBackupFile.ready");
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(2, 0, 2, 0)', "end processing ${strHistoryFile}, ${strBackupFile}");
$self->testResult(
sub {fileList($self->{strSpoolPath})}, "(${strHistoryFile}.ok, ${strBackupFile}.ok)",
sub {storageSpool()->list($self->{strSpoolPath})}, "(${strHistoryFile}.ok, ${strBackupFile}.ok)",
"${strHistoryFile}, ${strBackupFile} pushed");
$self->testResult(
sub {$self->{oFile}->exists(PATH_BACKUP_ARCHIVE, "$self->{strArchiveId}/${strHistoryFile}")}, true,
sub {storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/$self->{strArchiveId}/${strHistoryFile}")}, true,
"${strHistoryFile} in archive");
$self->testResult(
sub {$self->{oFile}->exists(PATH_BACKUP_ARCHIVE, "$self->{strArchiveId}/${strBackupFile}")}, true,
sub {storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/$self->{strArchiveId}/${strBackupFile}")}, true,
"${strBackupFile} in archive");
# Remove history and backup files
fileRemove("$self->{strWalPath}/${strHistoryFile}");
fileRemove("$self->{strWalStatusPath}/$strHistoryFile.ready");
fileRemove("$self->{strWalPath}/${strBackupFile}");
fileRemove("$self->{strWalStatusPath}/$strBackupFile.ready");
storageTest()->remove("$self->{strWalPath}/${strHistoryFile}");
storageTest()->remove("$self->{strWalStatusPath}/$strHistoryFile.ready");
storageTest()->remove("$self->{strWalPath}/${strBackupFile}");
storageTest()->remove("$self->{strWalStatusPath}/$strBackupFile.ready");
#---------------------------------------------------------------------------------------------------------------------------
# Generate a normal segment
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "processing ${strSegment}.gz");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
"${strSegment} WAL in archive");
# Remove the WAL and process so the .ok file is removed
@@ -557,24 +551,24 @@ sub run
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "remove ${strSegment}.ready");
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "${strSegment}.ok removed");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "${strSegment}.ok removed");
# Generate the same WAL again
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "processed duplicate ${strSegment}.gz");
$self->testResult(sub {fileList($self->{strSpoolPath})}, "${strSegment}.ok", "${strSegment} pushed");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "${strSegment}.ok", "${strSegment} pushed");
$self->testResult(
sub {fileStringRead("$self->{strSpoolPath}/${strSegment}.ok")},
sub {${storageSpool()->get("$self->{strSpoolPath}/${strSegment}.ok")}},
"0\nWAL segment ${strSegment} already exists in the archive with the same checksum\n" .
'HINT: this is valid in some recovery scenarios but may also indicate a problem.',
"${strSegment}.ok warning status");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
"${strSegment} WAL in archive");
# Remove the WAL
@@ -598,21 +592,21 @@ sub run
foreach my $strSegment (@strySegment)
{
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
}
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(3, 3, 1, 0)', "process and drop files");
$self->testResult(
sub {fileList($self->{strSpoolPath})}, '(' . join('.ok, ', @strySegment) . '.ok)',
sub {storageSpool()->list($self->{strSpoolPath})}, '(' . join('.ok, ', @strySegment) . '.ok)',
join(', ', @strySegment) . " ok drop files written");
foreach my $strSegment (@strySegment)
{
$self->testResult(
sub {fileStringRead("$self->{strSpoolPath}/${strSegment}.ok")},
$strSegment eq $strySegment[0] ? '' :
sub {${storageSpool()->get("$self->{strSpoolPath}/${strSegment}.ok")}},
$strSegment eq $strySegment[0] ? undef :
"0\ndropped WAL file ${strSegment} because archive queue exceeded " . optionGet(OPTION_ARCHIVE_QUEUE_MAX) .
' bytes',
"verify ${strSegment} status");
@@ -626,7 +620,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "final process to remove ok files");
$self->testResult(sub {fileList($self->{strSpoolPath})}, "[undef]", "ok files removed");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "ok files removed");
}
################################################################################################################################
@@ -634,6 +628,10 @@ sub run
{
my $oPush = new pgBackRest::Archive::ArchivePush($self->backrestExe());
$self->optionReset($oOption, OPTION_ARCHIVE_ASYNC);
$self->optionReset($oOption, OPTION_SPOOL_PATH);
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
@@ -656,12 +654,12 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->testResult(sub {$oPush->process("pg_xlog/${strSegment}")}, 0, "${strSegment} WAL pushed (with relative path)");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
"${strSegment} WAL in archive");
$self->walRemove($self->{strWalPath}, $strSegment);
@@ -672,11 +670,11 @@ sub run
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL dropped");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, '[undef]',
"${strSegment} WAL in archive");
# Set more realistic queue max and allow segment to push
@@ -685,7 +683,7 @@ sub run
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL pushed");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
"${strSegment} WAL in archive");
$self->walRemove($self->{strWalPath}, $strSegment);
@@ -702,8 +700,8 @@ sub run
# Write an error file and verify that it doesn't error the first time around
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
filePathCreate($self->{strSpoolPath}, undef, undef, true);
fileStringWrite("$self->{strSpoolPath}/${strSegment}.error", ERROR_ARCHIVE_TIMEOUT . "\ntest error");
storageTest()->pathCreate($self->{strSpoolPath}, {bCreateParent => true});
storageTest()->put("$self->{strSpoolPath}/${strSegment}.error", ERROR_ARCHIVE_TIMEOUT . "\ntest error");
$self->testException(
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_ARCHIVE_TIMEOUT,
@@ -712,31 +710,31 @@ sub run
$self->testResult($oPush->{bConfessOnError}, true, "went through error loop");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, '[undef]',
"${strSegment} WAL not in archive");
#---------------------------------------------------------------------------------------------------------------------------
# Write an OK file so the async process is not actually started
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
fileStringWrite("$self->{strSpoolPath}/${strSegment}.ok");
storageTest()->put("$self->{strSpoolPath}/${strSegment}.ok");
$self->testResult(
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0,
"${strSegment} WAL pushed async from synthetic ok file");
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, '[undef]',
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, '[undef]',
"${strSegment} WAL not in archive");
#---------------------------------------------------------------------------------------------------------------------------
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, 0, "${strSegment} WAL pushed async");
exit if ($iProcessId != $PID);
$self->testResult(
sub {walSegmentFind($self->{oFile}, $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
"${strSegment} WAL in archive");
$self->walRemove($self->{strWalPath}, $strSegment);
@@ -754,7 +752,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{oFile}, $self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->walGenerate($self->{strWalPath}, WAL_VERSION_94, 1, $strSegment);
$self->optionSetTest($oOption, OPTION_BACKUP_HOST, BOGUS);
$self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 60);
@@ -762,8 +760,8 @@ sub run
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
$self->testException(
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_HOST_CONNECT,
'remote process terminated on ' . BOGUS . ' host.*');
sub {$oPush->process("$self->{strWalPath}/${strSegment}")}, ERROR_FILE_READ,
"process '" . BOGUS . " remote' terminated.*");
exit if ($iProcessId != $PID);
# Disable async archiving
test/lib/pgBackRestTest/Module/Archive/ArchiveStopTest.pm
+21 -17
View File
@@ -21,9 +21,9 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
@@ -48,21 +48,23 @@ sub run
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version'))) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true});
my $oStorage = storageRepo();
# Create compression extension
my $strCompressExt = $bCompress ? ".$oFile->{strCompressExtension}" : '';
my $strCompressExt = $bCompress ? qw{.} . COMPRESS_EXT : '';
# Create the xlog path
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
filePathCreate($strXlogPath, undef, false, true);
$oStorage->pathCreate($strXlogPath, {bCreateParent => true});
# Create the test path for pg_control and copy pg_control for stanza-create
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
storageTest()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin',
$oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
# Create the archive info file
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . OPTION_ONLINE});
@@ -74,16 +76,16 @@ sub run
if ($iError == 0)
{
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
$oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
}
# Push two more segments with errors to exceed archive-max-mb
$oHostDbMaster->archivePush(
$strXlogPath, $strArchiveTestFile, 2, $iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
$strXlogPath, $strArchiveTestFile, 2, $iError ? ERROR_FILE_READ : ERROR_ARCHIVE_MISMATCH);
$oHostDbMaster->archivePush(
$strXlogPath, $strArchiveTestFile, 3, $iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
$strXlogPath, $strArchiveTestFile, 3, $iError ? ERROR_FILE_READ : ERROR_ARCHIVE_MISMATCH);
# Now this segment will get dropped
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 4);
@@ -91,13 +93,14 @@ sub run
# Fix the database version
if ($iError == 0)
{
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
}
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oFile->list(
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001', {strExpression => '^(?!000000010000000100000002).+'})},
sub {$oStorage->list(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001',
{strExpression => '^(?!000000010000000100000002).+'})},
"000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt}",
'segment 2-4 not pushed (2 is pushed sometimes when remote but ignore)', {iWaitSeconds => 5});
@@ -105,8 +108,9 @@ sub run
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile, 5);
$self->testResult(
sub {$oFile->list(
PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-1/0000000100000001', {strExpression => '^(?!000000010000000100000002).+'})},
sub {$oStorage->list(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001',
{strExpression => '^(?!000000010000000100000002).+'})},
"(000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt}, " .
"000000010000000100000005-72b9da071c13957fb4ca31f05dbd5c644297c2f7${strCompressExt})",
'segment 5 is pushed', {iWaitSeconds => 5});
test/lib/pgBackRestTest/Module/Archive/ArchiveUnitTest.pm
+27 -32
View File
@@ -11,14 +11,13 @@ use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::ArchiveCommon;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
@@ -95,58 +94,54 @@ sub run
################################################################################################################################
if ($self->begin("${strModule}::walSegmentFind()"))
{
my $strArchiveId = '9.4-1';
my $oFile = new pgBackRest::File(
$self->stanza(),
$self->testPath(),
new pgBackRest::Protocol::Common::Common(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
));
my $oOption = {};
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->testPath());
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
my $strArchivePath = $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveId);;
my $strArchiveId = '9.4-1';
my $strArchivePath = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalSegment = '000000010000000100000001ZZ';
$self->testException(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, ERROR_ASSERT, "${strWalSegment} is not a WAL segment");
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, ERROR_ASSERT,
"${strWalSegment} is not a WAL segment");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = '000000010000000100000001';
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, undef, "${strWalSegment} WAL not found");
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, undef, "${strWalSegment} WAL not found");
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment, .1)}, ERROR_ARCHIVE_TIMEOUT,
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, .1)}, ERROR_ARCHIVE_TIMEOUT,
"could not find WAL segment ${strWalSegment} after 0.1 second(s)");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalMajorPath = "${strArchivePath}/" . substr($strWalSegment, 0, 16);
my $strWalSegmentHash = "${strWalSegment}-53aa5d59515aa7288ae02ba414c009aed1ca73ad";
filePathCreate($strWalMajorPath, undef, false, true);
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, substr($strWalSegment, 8, 16))}, $strWalSegmentHash,
sub {walSegmentFind(storageRepo(), $strArchiveId, substr($strWalSegment, 8, 16))}, $strWalSegmentHash,
"${strWalSegment} WAL found without timeline");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalSegmentHash2 = "${strWalSegment}-a0b0d38b8aa263e25b8ff52a0a4ba85b6be97f9b.gz";
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash2}");
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash2}");
$self->testException(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, ERROR_ARCHIVE_DUPLICATE,
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, ERROR_ARCHIVE_DUPLICATE,
"duplicates found in archive for WAL segment ${strWalSegment}: ${strWalSegmentHash}, ${strWalSegmentHash2}");
#---------------------------------------------------------------------------------------------------------------------------
@@ -154,31 +149,31 @@ sub run
my $strWalSegmentHash3 = "${strWalSegment3}-dcdd09246e1918e88c67cf44b35edc23b803d879";
my $strWalMajorPath3 = "${strArchivePath}/" . substr($strWalSegment3, 0, 16);
filePathCreate($strWalMajorPath3, undef, false, true);
fileStringWrite("${strWalMajorPath3}/${strWalSegmentHash3}");
storageRepo()->pathCreate($strWalMajorPath3, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath3}/${strWalSegmentHash3}");
$self->testException(
sub {walSegmentFind($oFile, $strArchiveId, substr($strWalSegment, 8, 16))}, ERROR_ARCHIVE_DUPLICATE,
sub {walSegmentFind(storageRepo(), $strArchiveId, substr($strWalSegment, 8, 16))}, ERROR_ARCHIVE_DUPLICATE,
"duplicates found in archive for WAL segment XXXXXXXX" . substr($strWalSegment, 8, 16) .
": ${strWalSegmentHash}, ${strWalSegmentHash2}, ${strWalSegmentHash3}");
fileRemove("${strWalMajorPath}/${strWalSegmentHash}");
fileRemove("${strWalMajorPath3}/${strWalSegmentHash3}");
storageRepo()->remove("${strWalMajorPath}/${strWalSegmentHash}");
storageRepo()->remove("${strWalMajorPath3}/${strWalSegmentHash3}");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash2,
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash2,
"${strWalSegment} WAL found with compressed extension");
fileRemove("${strWalMajorPath}/${strWalSegmentHash2}");
storageRepo()->remove("${strWalMajorPath}/${strWalSegmentHash2}");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = $strWalSegment . '.partial';
$strWalSegmentHash = "${strWalSegment}-996195c807713ef9262170043e7222cb150aef70";
fileStringWrite("${strWalMajorPath}/${strWalSegmentHash}");
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(
sub {walSegmentFind($oFile, $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
}
}
test/lib/pgBackRestTest/Module/Backup/BackupInfoUnitTest.pm
+13 -20
View File
@@ -19,13 +19,11 @@ use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
@@ -39,11 +37,7 @@ sub initModule
{
my $self = shift;
$self->{strDbPath} = $self->testPath() . '/db';
$self->{strRepoPath} = $self->testPath() . '/repo';
$self->{strArchivePath} = "$self->{strRepoPath}/archive/" . $self->stanza();
$self->{strBackupPath} = "$self->{strRepoPath}/backup/" . $self->stanza();
$self->{strSpoolPath} = "$self->{strArchivePath}/out";
}
####################################################################################################################################
@@ -53,19 +47,17 @@ sub initTest
{
my $self = shift;
# Create archive info path
filePathCreate($self->{strArchivePath}, undef, true, true);
# Load options
my $oOption = {};
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->testPath() . '/repo');
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
# Create the local file object
$self->{oStorage} = storageRepo();
# Create backup info path
filePathCreate($self->{strBackupPath}, undef, true, true);
# Create pg_control path
filePathCreate(($self->{strDbPath} . '/' . DB_PATH_GLOBAL), undef, false, true);
# Copy a pg_control file into the pg_control path
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $self->{strDbPath} . '/' .
DB_FILE_PGCONTROL);
$self->{oStorage}->pathCreate(STORAGE_REPO_BACKUP, {bCreateParent => true});
}
####################################################################################################################################
@@ -79,7 +71,8 @@ sub run
################################################################################################################################
if ($self->begin("BackupInfo::confirmDb()"))
{
my $oBackupInfo = new pgBackRest::Backup::Info($self->{strBackupPath}, false, false);
my $oBackupInfo = new pgBackRest::Backup::Info($self->{oStorage}->pathGet(STORAGE_REPO_BACKUP), false, false,
{bIgnoreMissing => true});
$oBackupInfo->create(PG_VERSION_93, WAL_VERSION_93_SYS_ID, '937', '201306121', true);
my $strBackupLabel = "20170403-175647F";
test/lib/pgBackRestTest/Module/Backup/BackupUnitTest.pm
+29 -42
View File
@@ -12,6 +12,7 @@ use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Backup::Common;
use pgBackRest::Common::Exception;
@@ -19,11 +20,10 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Env::Host::HostBackupTest;
@@ -116,44 +116,31 @@ sub run
################################################################################################################################
if ($self->begin('backupLabel()'))
{
# Create the local file object
my $strRepoPath = $self->testPath() . '/repo';
my $oFile =
new pgBackRest::File
(
$self->stanza(),
$strRepoPath,
new pgBackRest::Protocol::Common::Common
(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
)
);
my $oOption = {};
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->testPath() . '/repo');
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH); logEnable();
#---------------------------------------------------------------------------------------------------------------------------
my $lTime = time();
my $strFullLabel = backupLabelFormat(BACKUP_TYPE_FULL, undef, $lTime);
$oFile->pathCreate(PATH_BACKUP_CLUSTER, $strFullLabel, undef, undef, true);
storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/${strFullLabel}", {bCreateParent => true});
my $strNewFullLabel = backupLabel($oFile, BACKUP_TYPE_FULL, undef, $lTime);
my $strNewFullLabel = backupLabel(storageRepo(), BACKUP_TYPE_FULL, undef, $lTime);
$self->testResult(sub {$strFullLabel ne $strNewFullLabel}, true, 'new full label <> existing full backup dir');
#---------------------------------------------------------------------------------------------------------------------------
executeTest('rmdir ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, $strFullLabel));
executeTest('rmdir ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strFullLabel}"));
$oFile->pathCreate(
PATH_BACKUP_CLUSTER, PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime), undef, undef, true);
fileStringWrite($oFile->pathGet(
PATH_BACKUP_CLUSTER,
PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime) .
"/${strFullLabel}.manifest.$oFile->{strCompressExtension}"));
storageRepo()->pathCreate(
STORAGE_REPO_BACKUP . qw(/) . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime), {bCreateParent => true});
storageRepo()->put(
STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime) .
"/${strFullLabel}.manifest." . COMPRESS_EXT);
$strNewFullLabel = backupLabel($oFile, BACKUP_TYPE_FULL, undef, $lTime);
$strNewFullLabel = backupLabel(storageRepo(), BACKUP_TYPE_FULL, undef, $lTime);
$self->testResult(sub {$strFullLabel ne $strNewFullLabel}, true, 'new full label <> existing full history file');
@@ -161,7 +148,7 @@ sub run
$lTime = time() + 1000;
$strFullLabel = backupLabelFormat(BACKUP_TYPE_FULL, undef, $lTime);
$strNewFullLabel = backupLabel($oFile, BACKUP_TYPE_FULL, undef, $lTime);
$strNewFullLabel = backupLabel(storageRepo(), BACKUP_TYPE_FULL, undef, $lTime);
$self->testResult(sub {$strFullLabel eq $strNewFullLabel}, true, 'new full label in future');
@@ -170,23 +157,23 @@ sub run
$strFullLabel = backupLabelFormat(BACKUP_TYPE_FULL, undef, $lTime);
my $strDiffLabel = backupLabelFormat(BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$oFile->pathCreate(PATH_BACKUP_CLUSTER, $strDiffLabel, undef, undef, true);
storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/${strDiffLabel}", {bCreateParent => true});
my $strNewDiffLabel = backupLabel($oFile, BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
my $strNewDiffLabel = backupLabel(storageRepo(), BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$self->testResult(sub {$strDiffLabel ne $strNewDiffLabel}, true, 'new diff label <> existing diff backup dir');
#---------------------------------------------------------------------------------------------------------------------------
executeTest('rmdir ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, $strDiffLabel));
executeTest('rmdir ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strDiffLabel}"));
$oFile->pathCreate(
PATH_BACKUP_CLUSTER, PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime), undef, true, true);
fileStringWrite($oFile->pathGet(
PATH_BACKUP_CLUSTER,
PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime) .
"/${strDiffLabel}.manifest.$oFile->{strCompressExtension}"));
storageRepo()->pathCreate(
STORAGE_REPO_BACKUP . qw(/) . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime),
{bIgnoreExists => true, bCreateParent => true});
storageRepo()->put(
STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime) .
"/${strDiffLabel}.manifest." . COMPRESS_EXT);
$strNewDiffLabel = backupLabel($oFile, BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$strNewDiffLabel = backupLabel(storageRepo(), BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$self->testResult(sub {$strDiffLabel ne $strNewDiffLabel}, true, 'new full label <> existing diff history file');
@@ -194,7 +181,7 @@ sub run
$lTime = time() + 1000;
$strDiffLabel = backupLabelFormat(BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$strNewDiffLabel = backupLabel($oFile, BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$strNewDiffLabel = backupLabel(storageRepo(), BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$self->testResult(sub {$strDiffLabel eq $strNewDiffLabel}, true, 'new diff label in future');
}
test/lib/pgBackRestTest/Module/Common/CommonIniTest.pm
+91 -47
View File
@@ -15,7 +15,6 @@ use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::FileCommon;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
@@ -100,56 +99,77 @@ sub run
################################################################################################################################
if ($self->begin("Ini->new()"))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {(new pgBackRest::Common::Ini($strTestFile, {bIgnoreMissing => true}))->exists()}, false, 'ignore missing');
#---------------------------------------------------------------------------------------------------------------------------
my $oIni = new pgBackRest::Common::Ini(
$strTestFile, {bLoad => false, iInitFormat => 4, strInitVersion => '1.01'});
$self->testResult($oIni->exists(), false, 'file does not exist');
$oIni->save();
$oIni->saveCopy();
$self->testResult($oIni->exists(), false, 'file does not exist after saveCopy()');
$self->testResult(
sub {fileStringRead($strTestFile)},
sub {${storageTest()->get($strTestFile . INI_COPY_EXT)}},
$self->iniHeader(undef, 4, '1.01', '488e5ca1a018cd7cd6d4e15150548f39f493dacd'),
'empty with synthetic format and version');
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite($strTestFile);
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_CONFIG, 'no key/value pairs found');
$oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
$oIni->saveCopy();
$self->testResult(
sub {${storageTest()->get($strTestFile . INI_COPY_EXT)}},
$self->iniHeader(undef, BACKREST_FORMAT, BACKREST_VERSION, $oIni->hash()),
'empty with default format and version');
#---------------------------------------------------------------------------------------------------------------------------
$oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
$self->testResult(
sub {storageTest()->list($self->testPath())},
'test.ini.copy',
'only copy is saved');
$oIni->save();
$self->testResult(
sub {fileStringRead($strTestFile)},
$self->iniHeader(undef, BACKREST_FORMAT, BACKREST_VERSION, $oIni->hash()),
'empty with default format and version');
sub {storageTest()->list($self->testPath())},
'(test.ini, test.ini.copy)',
'both versions are saved');
$self->testException(
sub {$oIni->saveCopy()}, ERROR_ASSERT,
"cannot save copy only when '${strTestFile}' exists");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'normal load');
#---------------------------------------------------------------------------------------------------------------------------
my $hIni = iniParse(fileStringRead($strTestFile));
my $hIni = iniParse(${storageTest()->get($strTestFile)});
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = BOGUS;
fileStringWrite($strTestFile, iniRender($hIni));
fileStringWrite($strTestFileCopy, iniRender($hIni));
storageTest()->put($strTestFile, iniRender($hIni));
storageTest()->put($strTestFileCopy, iniRender($hIni));
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_CHECKSUM,
"invalid checksum in '${strTestFile}', expected '" .
$oIni->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM) . "' but found 'bogus'");
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "invalid checksum in '${strTestFile}', expected '" .
# $oIni->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM) . "' but found 'bogus'");
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = $oIni->hash();
fileStringWrite($strTestFile, iniRender($hIni));
storageTest()->put($strTestFile, iniRender($hIni));
#---------------------------------------------------------------------------------------------------------------------------
$oIni->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, BACKREST_FORMAT - 1);
$oIni->save();
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FORMAT,
"invalid format in '${strTestFile}', expected " . BACKREST_FORMAT . ' but found ' . (BACKREST_FORMAT - 1));
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "invalid format in '${strTestFile}', expected " . BACKREST_FORMAT . ' but found ' . (BACKREST_FORMAT - 1));
$oIni->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, BACKREST_FORMAT);
$oIni->save();
@@ -159,17 +179,17 @@ sub run
$oIni->save();
$self->testResult(
sub {fileStringRead($strTestFile)},
sub {${storageTest()->get($strTestFile . INI_COPY_EXT)}},
$self->iniHeader($oIni, undef, '1.01'),
'verify old version was written');
$oIni = new pgBackRest::Common::Ini($strTestFile);
$self->testResult(sub {$oIni->get(INI_SECTION_BACKREST, INI_KEY_VERSION)}, BACKREST_VERSION, 'version is updated on load');
$self->testResult(sub {$oIni->save()}, true, 'save changes');
$oIni->save();
$self->testResult(
sub {fileStringRead($strTestFile)},
sub {${storageTest()->get($strTestFile . INI_COPY_EXT)}},
$self->iniHeader($oIni, undef, BACKREST_VERSION),
'verify version is updated on load');
@@ -177,65 +197,89 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false, strContent => fileStringRead($strTestFile)})},
sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false, strContent => ${storageTest()->get($strTestFile)}})},
'[object]', 'new() passing content as a string');
#---------------------------------------------------------------------------------------------------------------------------
executeTest("rm -rf ${strTestFile}*");
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_UNKNOWN,
"unable to open ${strTestFile}");
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite($strTestFileCopy, BOGUS);
storageTest()->put($strTestFileCopy, BOGUS);
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFileCopy)}, ERROR_CONFIG,
"key/value pair 'bogus' found outside of a section");
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "key/value pair 'bogus' found outside of a section");
#---------------------------------------------------------------------------------------------------------------------------
$oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
my $oIniSource = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
fileStringWrite($strTestFileCopy, iniRender($oIni->{oContent}));
storageTest()->put($strTestFileCopy, iniRender($oIniSource->{oContent}));
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFileCopy)}, ERROR_CHECKSUM,
"invalid checksum in '${strTestFileCopy}', expected '" .
$oIni->hash() . "' but found [undef]");
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "invalid checksum in '${strTestFileCopy}', expected '" .
# $oIniSource->hash() . "' but found [undef]");
#---------------------------------------------------------------------------------------------------------------------------
$oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
$oIni->{oContent}->{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = 0;
$oIniSource = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
$oIniSource->save();
$self->testResult(
sub {$oIni->headerCheck({bIgnoreInvalid => true})}, false,
'ignore invalid header');
storageTest()->put($strTestFile, BOGUS);
# main invalid, copy ok
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'invalid main - load copy');
#---------------------------------------------------------------------------------------------------------------------------
fileRemove($strTestFileCopy);
storageTest()->remove($strTestFileCopy);
fileStringWrite($strTestFile, "[section]\n" . BOGUS);
storageTest()->put($strTestFile, "[section]\n" . BOGUS);
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_CONFIG, "unable to find '=' in 'bogus'");
# main invalid, copy missing
$self->testException
(sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
#"unable to find '=' in 'bogus'");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {iniParse("[section]\n" . BOGUS, {bIgnoreInvalid => true})}, undef, 'ignore invalid content');
storageTest()->put($strTestFile, iniRender($oIniSource->{oContent}));
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'main ok, copy missing');
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite($strTestFile, BOGUS);
storageTest()->put($strTestFileCopy, BOGUS);
# main ok, copy invalid
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'main ok, copy invalid');
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strTestFile, BOGUS);
# main invalid, copy invalid
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_CONFIG, "key/value pair 'bogus' found outside of a section");
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "key/value pair 'bogus' found outside of a section");
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite($strTestFile, iniRender($hIni));
storageTest()->put($strTestFile, iniRender($hIni));
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = BOGUS;
fileStringWrite($strTestFileCopy, iniRender($hIni));
storageTest()->put($strTestFileCopy, iniRender($hIni));
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'invalid header - load main');
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'invalid copy header - load main');
#---------------------------------------------------------------------------------------------------------------------------
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = $oIni->hash();
storageTest()->put($strTestFileCopy, iniRender($hIni));
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = BOGUS;
storageTest()->put($strTestFile, iniRender($hIni));
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'invalid main header - load copy');
}
################################################################################################################################
test/lib/pgBackRestTest/Module/Common/CommonIoBufferedTest.pm
+259
View File
@@ -0,0 +1,259 @@
####################################################################################################################################
# CommonIoBufferedTest.pm - tests for Common::Io::Buffered module
####################################################################################################################################
package pgBackRestTest::Module::Common::CommonIoBufferedTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use IO::Socket::UNIX;
use Time::HiRes qw(usleep);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# socketServer
####################################################################################################################################
sub socketServer
{
my $self = shift;
my $strSocketFile = shift;
my $fnServer = shift;
# Fork off the server
if (fork() == 0)
{
# Open the domain socket
my $oSocketServer = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Local => $strSocketFile, Listen => 1);
&log(INFO, " * socket server open");
# Wait for a connection and create IO object
my $oConnection = $oSocketServer->accept();
my $oIoHandle = new pgBackRest::Common::Io::Handle('socket server', $oConnection, $oConnection);
&log(INFO, " * socket server connected");
# Run server function
$fnServer->($oIoHandle);
# Shutdown server
$oSocketServer->close();
&log(INFO, " * socket server closed");
unlink($strSocketFile);
exit 0;
}
# Wait for client socket
my $oWait = waitInit(5);
while (!-e $strSocketFile && waitMore($oWait)) {};
# Open the client socket
my $oClient = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Peer => $strSocketFile);
if (!defined($oClient))
{
logErrorResult(ERROR_FILE_OPEN, 'unable to open client socket', $OS_ERROR);
}
&log(INFO, " * socket client connected");
return $oClient;
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strSocketFile = $self->testPath() . qw{/} . 'domain.socket';
################################################################################################################################
if ($self->begin('new() & timeout() & bufferMax()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoBuffered = $self->testResult(
sub {new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('test'), 10, 2048)}, '[object]', 'new - no handles');
$self->testResult(sub {$oIoBuffered->timeout()}, 10, ' check timeout');
$self->testResult(sub {$oIoBuffered->bufferMax()}, 2048, ' check buffer max');
}
################################################################################################################################
if ($self->begin('readLine() & writeLine()'))
{
my $oClient = $self->socketServer($strSocketFile, sub
{
my $oIoHandle = shift;
my $tResponse = '';
my $tData;
# Ack after timeout
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write 8 char line
$tResponse = '';
$tData = "12345678\n";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write 3 lines
$tResponse = '';
$tData = "1\n2\n345678\n";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write blank line
$tResponse = '';
$tData = "\n";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write char with no linefeed
$tResponse = '';
$tData = "A";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write linefeed
$tResponse = '';
$tData = "\n";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
});
#---------------------------------------------------------------------------------------------------------------------------
my $oIoBuffered = $self->testResult(
sub {new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('socket client', $oClient, $oClient), 1, 4)}, '[object]', 'open');
$self->testException(
sub {$oIoBuffered->readLine()}, ERROR_FILE_READ, 'unable to read line after 1 second(s) from socket client');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine()}, '12345678', 'read 8 char line');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine()}, '1', 'read 1 char line');
$self->testResult(sub {$oIoBuffered->readLine()}, '2', 'read 1 char line');
$self->testResult(sub {$oIoBuffered->readLine()}, '345678', 'read 6 char line');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine()}, '', 'read blank line');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine(undef, false)}, undef, 'read ignoring error');
$self->testException(
sub {$oIoBuffered->readLine(undef, true)}, ERROR_FILE_READ,
'unable to read line after 1 second(s) from socket client');
# Clear buffer so EOF tests pass
$oIoBuffered->writeLine();
$oIoBuffered->readLine();
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oIoBuffered->readLine()}, ERROR_FILE_READ, 'unexpected EOF reading line from socket client');
$self->testException(
sub {$oIoBuffered->readLine(false)}, ERROR_FILE_READ, 'unexpected EOF reading line from socket client');
$self->testResult(sub {$oIoBuffered->readLine(true)}, undef, 'ignore EOF');
}
################################################################################################################################
if ($self->begin('read'))
{
my $tBuffer;
my $oClient = $self->socketServer($strSocketFile, sub
{
my $oIoHandle = shift;
my $tResponse = '';
my $tData;
# Ack after timeout
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write mixed buffer
$tResponse = '';
$tData = "123\n123\n1";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write buffer
$tResponse = '';
$tData = "23456789";
$oIoHandle->write(\$tData);
# Wait before writing the rest to force client to loop
usleep(500);
$tData = "0";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
});
#---------------------------------------------------------------------------------------------------------------------------
my $oIoBuffered = $self->testResult(
sub {new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('socket client', $oClient, $oClient), 1, 8)}, '[object]', 'open');
$self->testException(
sub {$oIoBuffered->read(\$tBuffer, 1, true)}, ERROR_FILE_READ,
'unable to read 1 byte(s) after 1 second(s) from socket client');
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 1)}, 0, ' read 0 char buffer');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine()}, '123', ' read 3 char line');
undef($tBuffer);
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 1)}, 1, ' read 1 char buffer');
$self->testResult(sub {$tBuffer}, '1', ' check 1 char buffer');
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 3)}, 3, ' read 3 char buffer');
$self->testResult(sub {$tBuffer}, "123\n", ' check 3 char buffer');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
undef($tBuffer);
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 10)}, 10, ' read 10 char buffer');
$self->testResult(sub {$tBuffer}, '1234567890', ' check 10 char buffer');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 5)}, 0, ' expect EOF');
$self->testException(
sub {$oIoBuffered->read(\$tBuffer, 5, true)}, ERROR_FILE_READ,
'unable to read 5 byte(s) due to EOF from socket client');
}
}
1;
test/lib/pgBackRestTest/Module/Common/CommonIoHandleTest.pm
+216
View File
@@ -0,0 +1,216 @@
####################################################################################################################################
# CommonIoHandleTest.pm - tests for Common::Io::Handle module
####################################################################################################################################
package pgBackRestTest::Module::Common::CommonIoHandleTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC O_NONBLOCK);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Base;
use pgBackRest::Common::Io::Handle;
use pgBackRest::Common::Log;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strFile = $self->testPath() . qw{/} . 'file.txt';
my $strFileContent = 'TESTDATA';
my $iFileLength = length($strFileContent);
my $iFileLengthHalf = int($iFileLength / 2);
################################################################################################################################
if ($self->begin('new() & handleRead() & handleReadSet() & handleWrite() & handleWriteSet()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, undef)}, '[object]', 'new - no handles');
$self->testResult(sub {$oIoHandle->id()}, 'test', ' check error msg');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oIoHandle->handleReadSet(1)}, 1, ' set read handle');
$self->testResult(sub {$oIoHandle->handleRead()}, 1, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, undef, ' check write handle');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oIoHandle->handleWriteSet(2)}, 2, ' set write handle');
$self->testResult(sub {$oIoHandle->handleRead()}, 1, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, 2, ' check write handle');
#---------------------------------------------------------------------------------------------------------------------------
$oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', 1, undef)}, '[object]', 'new - read handle');
$self->testResult(sub {$oIoHandle->handleRead()}, 1, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, undef, ' check write handle');
#---------------------------------------------------------------------------------------------------------------------------
$oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, 2)}, '[object]', 'new - write handle');
$self->testResult(sub {$oIoHandle->handleRead()}, undef, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, 2, ' check write handle');
#---------------------------------------------------------------------------------------------------------------------------
$oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', 1, 2)}, '[object]', 'new - read/write handle');
$self->testResult(sub {$oIoHandle->handleRead()}, 1, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, 2, ' check write handle');
}
################################################################################################################################
if ($self->begin('read()'))
{
my $tContent;
my $fhRead;
#---------------------------------------------------------------------------------------------------------------------------
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
sysopen($fhRead, $strFile, O_RDONLY)
or confess &log(ERROR, "unable to open '${strFile}'");
my $oIoHandle = $self->testResult(sub {new pgBackRest::Common::Io::Handle("'$strFile'", $fhRead)}, '[object]', 'open');
$self->testException(
sub {$oIoHandle->read(\$tContent, -1)}, ERROR_FILE_READ, "unable to read from '${strFile}': Negative length");
#---------------------------------------------------------------------------------------------------------------------------
$tContent = undef;
sysopen($fhRead, $strFile, O_RDONLY)
or confess &log(ERROR, "unable to open '${strFile}'");
$oIoHandle = $self->testResult(sub {new pgBackRest::Common::Io::Handle("'$strFile'", $fhRead)}, '[object]', 'open');
$self->testResult(sub {$oIoHandle->read(\$tContent, $iFileLengthHalf)}, $iFileLengthHalf, ' read part 1');
$self->testResult($tContent, substr($strFileContent, 0, $iFileLengthHalf), ' check read');
$self->testResult(
sub {$oIoHandle->read(
\$tContent, $iFileLength - $iFileLengthHalf)}, $iFileLength - $iFileLengthHalf, ' read part 2');
$self->testResult($tContent, $strFileContent, ' check read');
$self->testResult(sub {$oIoHandle->read(\$tContent, 1)}, 0, ' eof');
}
################################################################################################################################
if ($self->begin('write()'))
{
my $tContent;
my $fhRead;
my $fhWrite;
#---------------------------------------------------------------------------------------------------------------------------
sysopen($fhWrite, $strFile, O_WRONLY | O_CREAT | O_TRUNC)
or confess &log(ERROR, "unable to open '${strFile}'");
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle("'$strFile'", undef, $fhWrite)}, '[object]', 'open write');
$self->testException(
sub {$oIoHandle->write(undef)}, ERROR_FILE_WRITE,
"unable to write to '${strFile}': Can't use an undefined value as a SCALAR reference");
#---------------------------------------------------------------------------------------------------------------------------
$tContent = substr($strFileContent, 0, $iFileLengthHalf);
$self->testResult(sub {$oIoHandle->write(\$tContent)}, $iFileLengthHalf, ' write part 1');
$self->testResult(sub {$oIoHandle->size()}, $iFileLengthHalf, ' check part 1 size');
$tContent = substr($strFileContent, $iFileLengthHalf);
$self->testResult(sub {$oIoHandle->write(\$tContent)}, $iFileLength - $iFileLengthHalf, ' write part 2');
$self->testResult(sub {$oIoHandle->size()}, $iFileLength, ' check part 2 size');
$self->testResult(sub {$oIoHandle->close()}, true, ' close');
sysopen($fhRead, $strFile, O_RDONLY)
or confess &log(ERROR, "unable to open '${strFile}'");
$oIoHandle = $self->testResult(sub {new pgBackRest::Common::Io::Handle("'$strFile'", $fhRead)}, '[object]', 'open read');
$tContent = undef;
$self->testResult(sub {$oIoHandle->read(\$tContent, $iFileLength)}, $iFileLength, ' read');
$self->testResult($tContent, $strFileContent, ' check write content');
}
################################################################################################################################
if ($self->begin('result() & resultSet()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, undef)}, '[object]', 'new - no handles');
$self->testResult(sub {$oIoHandle->resultSet('Module::1', 1)}, 1, ' set int result');
$self->testResult(sub {$oIoHandle->resultSet('Module::2', {value => 2})}, '{value => 2}', ' set hash result');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoHandle->result('Module::1')}, 1, ' check int result');
$self->testResult(sub {$oIoHandle->result('Module::2')}, '{value => 2}', ' check hash result');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoHandle->{rhResult}}, '{Module::1 => 1, Module::2 => {value => 2}}', ' check all results');
}
################################################################################################################################
if ($self->begin('isA()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, undef)}, '[object]', 'new - no handles');
$self->testResult(
sub {$oIoHandle->isA()}, '(' . COMMON_IO_HANDLE . ', ' . COMMON_IO_BASE . ')', ' check isA');
}
################################################################################################################################
if ($self->begin('className()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, undef)}, '[object]', 'new - no handles');
$self->testResult(sub {$oIoHandle->className()}, COMMON_IO_HANDLE, ' check class name');
}
################################################################################################################################
if ($self->begin('close()'))
{
my $tContent;
my $fhRead;
#---------------------------------------------------------------------------------------------------------------------------
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
sysopen($fhRead, $strFile, O_RDONLY)
or confess &log(ERROR, "unable to open '${strFile}'");
my $oIoHandle = $self->testResult(sub {new pgBackRest::Common::Io::Handle("'$strFile'", $fhRead)}, '[object]', 'open read');
$self->testResult(sub {$oIoHandle->read(\$tContent, $iFileLengthHalf)}, $iFileLengthHalf, ' read');
$self->testResult(sub {$oIoHandle->close()}, true, ' close');
$self->testResult(sub {$oIoHandle->result(COMMON_IO_HANDLE)}, $iFileLengthHalf, ' check result');
# Destroy and to make sure close runs again
undef($oIoHandle);
}
}
1;
test/lib/pgBackRestTest/Module/File/FileLinkTest.pm → test/lib/pgBackRestTest/Module/Common/CommonIoProcessTest.pm
+25 -24
View File
@@ -1,8 +1,8 @@
####################################################################################################################################
# FileLinkTest.pm - Tests for FileCommon::fileLinkDestination
# CommonIoProcessTest.pm - tests for Common::Io::Process module
####################################################################################################################################
package pgBackRestTest::Module::File::FileLinkTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
package pgBackRestTest::Module::Common::CommonIoProcessTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
@@ -12,12 +12,10 @@ use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Fcntl qw(:mode);
use File::stat;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Io::Process;
use pgBackRest::Common::Log;
use pgBackRest::FileCommon;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
@@ -29,29 +27,32 @@ sub run
{
my $self = shift;
# Test data
my $strFile = $self->testPath() . qw{/} . 'file.txt';
my $strFileContent = 'TESTDATA';
################################################################################################################################
if ($self->begin("FileCommon::fileLinkDestination()"))
if ($self->begin('new() & processId()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strTestLink = $self->testPath() . '/public_dir_link';
$self->testException(
sub {fileLinkDestination($strTestLink)}, ERROR_FILE_MISSING,
"unable to get destination for link ${strTestLink}: No such file or directory");
my $oIoProcess = $self->testResult(sub {
new pgBackRest::Common::Io::Process(
new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('test'), 1, 32), "echo '${strFileContent}'")}, '[object]', 'new - echo');
$self->testResult(sub {defined($oIoProcess->processId())}, true, ' process id defined');
}
################################################################################################################################
if ($self->begin('close() & error()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strTestPath = $self->testPath() . '/public_dir';
filePathCreate($strTestPath);
my $oIoProcess =
new pgBackRest::Common::Io::Process(
new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('test'), 1, 32), "echo '${strFileContent}'");
$oIoProcess->close();
$self->testException(
sub {fileLinkDestination($strTestPath)}, ERROR_FILE_OPEN,
"unable to get destination for link ${strTestPath}: Invalid argument");
#---------------------------------------------------------------------------------------------------------------------------
symlink($strTestPath, $strTestLink)
or confess &log(ERROR, "unable to create symlink from ${strTestPath} to ${strTestLink}");
$self->testResult(sub {fileLinkDestination($strTestLink)}, $strTestPath, 'get link destination');
sub {$oIoProcess->error()}, ERROR_ASSERT, 'cannot call error() after process has been closed');
}
}
test/lib/pgBackRestTest/Module/Config/ConfigConfigTest.pm
+20 -21
View File
@@ -15,7 +15,6 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::FileCommon;
use pgBackRestTest::Common::RunTest;
@@ -72,7 +71,7 @@ sub run
{
$oConfig = {};
$$oConfig{$self->stanza() . ':' . &CMD_BACKUP}{&OPTION_PROCESS_MAX} = 2;
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -86,7 +85,7 @@ sub run
{
$oConfig = {};
$$oConfig{$self->stanza()}{&OPTION_PROCESS_MAX} = 3;
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -100,7 +99,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL . ':' . &CMD_BACKUP}{'thread-max'} = 2;
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -114,7 +113,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL}{&OPTION_PROCESS_MAX} = 5;
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -127,7 +126,7 @@ sub run
if ($self->begin('default - option ' . OPTION_PROCESS_MAX))
{
$oConfig = {};
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -141,7 +140,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL}{&OPTION_PROCESS_MAX} = 9;
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -156,7 +155,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL . ':' . &CMD_BACKUP}{&OPTION_HARDLINK} = 'Y';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -169,7 +168,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_CONSOLE} = BOGUS;
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -182,7 +181,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_CONSOLE} = lc(INFO);
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -203,7 +202,7 @@ sub run
{
$oConfig = {};
$$oConfig{$self->stanza() . ':' . &CMD_EXPIRE}{&OPTION_RETENTION_FULL} = 2;
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_CONFIG, $strConfigFile);
@@ -216,7 +215,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL . ':' . &CMD_BACKUP}{&OPTION_COMPRESS} = 'n';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -230,7 +229,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL . ':' . &CMD_RESTORE}{&OPTION_RESTORE_RECOVERY_OPTION} = 'bogus=';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -243,7 +242,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL . ':' . &CMD_RESTORE}{&OPTION_RESTORE_RECOVERY_OPTION} = '=bogus';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -257,7 +256,7 @@ sub run
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL . ':' . &CMD_RESTORE}{&OPTION_RESTORE_RECOVERY_OPTION} =
'archive-command=/path/to/pgbackrest';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -271,7 +270,7 @@ sub run
{
$oConfig = {};
$$oConfig{$self->stanza()}{&OPTION_RESTORE_RECOVERY_OPTION} = ['standby-mode=on', 'a=b'];
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -286,7 +285,7 @@ sub run
{
$oConfig = {};
$$oConfig{$self->stanza()}{&OPTION_DB_PATH} = '/path/to/db';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_CONFIG, $strConfigFile);
@@ -299,7 +298,7 @@ sub run
{
$oConfig = {};
$$oConfig{$self->stanza()}{&OPTION_DB_PATH} = '/path/to/db';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_CONFIG, $strConfigFile);
@@ -314,7 +313,7 @@ sub run
{
$oConfig = {};
$$oConfig{$self->stanza()}{&OPTION_DB_PATH} = '/path/to/db';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_CONFIG, $strConfigFile);
@@ -327,7 +326,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_PATH} = '/repo';
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
@@ -341,7 +340,7 @@ sub run
{
$oConfig = {};
$$oConfig{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_PATH} = ['/repo', '/repo2'];
fileStringWrite($strConfigFile, iniRender($oConfig, true));
storageTest()->put($strConfigFile, iniRender($oConfig, true));
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, '/db');
test/lib/pgBackRestTest/Module/Config/ConfigOptionTest.pm
+3 -3
View File
@@ -220,12 +220,12 @@ sub run
$self->configLoadExpect($oOption, CMD_RESTORE);
}
if ($self->begin('invalid ' . OPTION_BUFFER_SIZE))
if ($self->begin('invalid ' . OPTION_COMPRESS_LEVEL))
{
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_BUFFER_SIZE, '512');
$self->optionSetTest($oOption, OPTION_COMPRESS_LEVEL, '12');
$self->configLoadExpect($oOption, CMD_RESTORE, ERROR_OPTION_INVALID_RANGE, '512', OPTION_BUFFER_SIZE);
$self->configLoadExpect($oOption, CMD_RESTORE, ERROR_OPTION_INVALID_RANGE, '12', OPTION_COMPRESS_LEVEL);
}
if ($self->begin(CMD_BACKUP . ' invalid value ' . OPTION_RETENTION_ARCHIVE_TYPE))
test/lib/pgBackRestTest/Module/Config/ConfigUnitTest.pm
+1 -1
View File
@@ -28,7 +28,7 @@ sub run
my $oConfig = {};
my $strConfigFile = $self->testPath() . '/pgbackrest.conf';
optionSet(OPTION_CONFIG, $strConfigFile, true);
commandSet(CMD_LOCAL);
commandSet(CMD_ARCHIVE_GET);
if ($self->begin('Config::configFileValidate()'))
{
test/lib/pgBackRestTest/Module/Expire/ExpireExpireTest.pm
+12 -13
View File
@@ -23,14 +23,13 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Expire;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Env::ExpireEnvTest;
use pgBackRestTest::Env::HostEnvTest;
####################################################################################################################################
# initStanzaOption
@@ -68,14 +67,14 @@ sub run
if ($self->begin("local"))
{
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(true, $self->expect());
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(true, $self->expect());
$self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath});
$self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE);
# Create the test object
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest($oHostBackup, $self->backrestExe(), $oFile, $self->expect(),
$self);
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest(
$oHostBackup, $self->backrestExe(), storageRepo(), $self->expect(), $self);
$oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_92);
@@ -165,14 +164,14 @@ sub run
if ($self->begin("Expire::stanzaUpgrade"))
{
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(true, $self->expect());
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(true, $self->expect());
$self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath});
$self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE);
# Create the test object
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest($oHostBackup, $self->backrestExe(), $oFile, $self->expect(),
$self);
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest(
$oHostBackup, $self->backrestExe(), storageRepo(), $self->expect(), $self);
$oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_92);
@@ -226,7 +225,7 @@ sub run
my $oExpire = new pgBackRest::Expire();
# Mismatched version
$oHostBackup->infoMunge($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
$oHostBackup->infoMunge(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB =>
{&INFO_ARCHIVE_KEY_DB_VERSION => PG_VERSION_93, &INFO_ARCHIVE_KEY_DB_SYSTEM_ID => WAL_VERSION_95_SYS_ID},
&INFO_ARCHIVE_SECTION_DB_HISTORY =>
@@ -239,10 +238,10 @@ sub run
"HINT: has a stanza-upgrade been performed?");
# Restore the info file
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
# Mismatched system ID
$oHostBackup->infoMunge($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
$oHostBackup->infoMunge(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB =>
{&INFO_ARCHIVE_KEY_DB_SYSTEM_ID => 6999999999999999999},
&INFO_ARCHIVE_SECTION_DB_HISTORY =>
@@ -255,7 +254,7 @@ sub run
"HINT: has a stanza-upgrade been performed?");
# Restore the info file
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
}
}
test/lib/pgBackRestTest/Module/File/FileCommonTest.pm
-125
View File
@@ -1,125 +0,0 @@
####################################################################################################################################
# FileCommonTest.pm - Common code for File tests
####################################################################################################################################
package pgBackRestTest::Module::File::FileCommonTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Remote::Master;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# initModule
#
# Common objects and variables used by all tests.
####################################################################################################################################
sub initModule
{
my $self = shift;
# Create the repo path so the remote won't complain that it's missing
my $strRepoPath = $self->testPath() . '/repo';
mkdir($strRepoPath, oct('0770'))
or confess "Unable to create repo directory: ${strRepoPath}";
# Create local
$self->{oLocal} = new pgBackRest::Protocol::Common::Common(
262144,
1,
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK,
HOST_PROTOCOL_TIMEOUT);
# Create remote
$self->{oRemote} = new pgBackRest::Protocol::Remote::Master(
BACKUP,
OPTION_DEFAULT_CMD_SSH,
$self->backrestExeOriginal() . ' --stanza=' . $self->stanza() .
" --type=backup --repo-path=${strRepoPath} --no-config --command=test remote",
262144,
OPTION_DEFAULT_COMPRESS_LEVEL,
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK,
$self->host(),
$self->backrestUser(),
HOST_PROTOCOL_TIMEOUT);
rmdir($strRepoPath)
or confess "Unable to remove repo directory: ${strRepoPath}";
}
####################################################################################################################################
# cleanModule
#
# Close objects created for tests.
####################################################################################################################################
sub cleanModule
{
my $self = shift;
$self->remote()->close();
}
####################################################################################################################################
# setup
#
# Setup directories for file tests.
####################################################################################################################################
sub setup
{
my $self = shift;
my $bRemote = shift;
my $bPrivate = shift;
# Remove the backrest private directory
if (fileExists($self->testPath() . '/private'))
{
executeTest(
'ssh ' . $self->backrestUser() . '\@' . $self->host() . ' rm -rf ' . $self->testPath() . '/private',
{bSuppressStdErr => true});
}
# Remove contents of the test directory
executeTest('rm -rf ' . $self->testPath() . '/*');
# Create the private directories
if (defined($bPrivate) && $bPrivate)
{
executeTest(
'ssh ' . $self->backrestUser() . '\@' . $self->host() . ' mkdir -m 700 ' . $self->testPath() . '/backrest_private',
{bSuppressStdErr => true});
executeTest('mkdir -m 700 ' . $self->testPath() . '/user_private');
}
# Create the file object
my $oFile = new pgBackRest::File
(
$self->stanza(),
$self->testPath(),
$bRemote ? $self->remote() : $self->local()
);
return $oFile;
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub host {return '127.0.0.1'}
sub local {return shift->{oLocal}}
sub remote {return shift->{oRemote}}
1;
test/lib/pgBackRestTest/Module/File/FileCompressTest.pm
-101
View File
@@ -1,101 +0,0 @@
####################################################################################################################################
# FileCompressTest.pm - Tests for File->compress()
####################################################################################################################################
package pgBackRestTest::Module::File::FileCompressTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote ??? enable remote tests and have them throw an error
foreach my $bRemote (false)
{
# Loop through exists
foreach my $bExists (false, true)
{
# Loop through error
foreach my $bError (false, true)
{
if (!$self->begin("rmt ${bRemote}, exists ${bExists}, err ${bError}")) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote, $bError);
my $strFile = $self->testPath() . '/test.txt';
my $strSourceHash;
my $iSourceSize;
if ($bError)
{
$strFile = $self->testPath() . '/' . ($bRemote ? 'user' : 'backrest') . '_private/test.txt';
}
elsif ($bExists)
{
executeTest("echo 'TESTDATA' > ${strFile}");
($strSourceHash, $iSourceSize) = $oFile->hashSize(PATH_BACKUP_ABSOLUTE, $strFile);
}
# Execute in eval in case of error
eval
{
$oFile->compress(PATH_BACKUP_ABSOLUTE, $strFile);
return true;
}
or do
{
if (!$bExists || $bError)
{
next;
}
confess $EVAL_ERROR;
};
if (!$bExists || $bError)
{
confess 'expected error';
}
my $strDestinationFile = $strFile . '.gz';
if (-e $strFile)
{
confess 'source file still exists';
}
unless (-e $strDestinationFile)
{
confess 'file was not compressed';
}
executeTest("gzip -d ${strDestinationFile}");
my ($strDestinationHash, $iDestinationSize) = $oFile->hashSize(PATH_BACKUP_ABSOLUTE, $strFile);
if ($strSourceHash ne $strDestinationHash)
{
confess "source ${strSourceHash} and destination ${strDestinationHash} file hashes do not match";
}
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FileCopyTest.pm
-244
View File
@@ -1,244 +0,0 @@
####################################################################################################################################
# FileCopyTest.pm - Tests for File->copy()
####################################################################################################################################
package pgBackRestTest::Module::File::FileCopyTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through small/large
for (my $iLarge = 0; $iLarge <= 3; $iLarge++)
{
# Loop through possible remotes
foreach my $strRemote ($iLarge ? (undef, 'db') : (undef, 'backup', 'db'))
{
# Loop through source path types
foreach my $strSourcePathType ($iLarge ? (PATH_BACKUP_ABSOLUTE) : (PATH_BACKUP_ABSOLUTE, PATH_DB_ABSOLUTE))
{
# Loop through destination path types
foreach my $strDestinationPathType ($iLarge ? (PATH_DB_ABSOLUTE) : (PATH_BACKUP_ABSOLUTE, PATH_DB_ABSOLUTE))
{
# Loop through source missing/present
foreach my $bSourceMissing ($iLarge ? (false) : (false, true))
{
# Loop through source ignore/require
foreach my $bSourceIgnoreMissing ($bSourceMissing ? (false, true) : (false))
{
# Loop through checksum append
foreach my $bChecksumAppend ($bSourceMissing || $iLarge ? (false) : (false, true))
{
# Loop through source compression
foreach my $bSourceCompressed ($bSourceMissing ? (false) : (false, true))
{
# Loop through destination compression
foreach my $bDestinationCompress ($bSourceMissing ? (false) : (false, true))
{
my $strSourcePath = $strSourcePathType eq PATH_DB_ABSOLUTE ? 'db' : 'backup';
my $strDestinationPath = $strDestinationPathType eq PATH_DB_ABSOLUTE ? 'db' : 'backup';
if (!$self->begin(
"lrg ${iLarge}, rmt " .
(defined($strRemote) && ($strRemote eq $strSourcePath || $strRemote eq $strDestinationPath) ? 1 : 0) .
', srcpth ' . (defined($strRemote) && $strRemote eq $strSourcePath ? 'rmt' : 'lcl') .
":${strSourcePath}, srcmiss ${bSourceMissing}, srcignmiss ${bSourceIgnoreMissing}, srccmp $bSourceCompressed, " .
'dstpth ' . (defined($strRemote) && $strRemote eq $strDestinationPath ? 'rmt' : 'lcl') .
":${strDestinationPath}, chkapp ${bChecksumAppend}, dstcmp $bDestinationCompress")) {next}
# Setup test directory and get file object
my $oFile = $self->setup(defined($strRemote), false);
executeTest('mkdir ' . $self->testPath() . '/backup');
executeTest('mkdir ' . $self->testPath() . '/db');
my $strSourceFile = $self->testPath() . "/${strSourcePath}/test-source";
my $strDestinationFile = $self->testPath() . "/${strDestinationPath}/test-destination";
my $strCopyHash;
my $iCopySize;
# Create the compressed or uncompressed test file
my $strSourceHash;
my $iSourceSize;
if (!$bSourceMissing)
{
if ($iLarge)
{
$strSourceFile .= '.bin';
$strDestinationFile .= '.bin';
if ($iLarge < 3)
{
executeTest('cp ' . $self->dataPath() . "/filecopy.archive${iLarge}.bin ${strSourceFile}");
}
else
{
for (my $iTableSizeIdx = 0; $iTableSizeIdx < 25; $iTableSizeIdx++)
{
executeTest('cat ' . $self->dataPath() . "/filecopy.table.bin >> ${strSourceFile}");
}
}
}
else
{
$strSourceFile .= '.txt';
$strDestinationFile .= '.txt';
system("echo 'TESTDATA' > ${strSourceFile}");
}
if ($iLarge == 1)
{
$strSourceHash = '4518a0fdf41d796760b384a358270d4682589820';
$iSourceSize = 16777216;
}
elsif ($iLarge == 2)
{
$strSourceHash = '1c7e00fd09b9dd11fc2966590b3e3274645dd031';
$iSourceSize = 16777216;
}
elsif ($iLarge == 3)
{
$strSourceHash = 'b2055a6ba15bf44359c18fbbf23c68b50a670eb0';
$iSourceSize = 26214400;
}
else
{
$strSourceHash = '06364afe79d801433188262478a76d19777ef351';
$iSourceSize = 9;
}
if ($bSourceCompressed)
{
system("gzip ${strSourceFile}");
$strSourceFile .= '.gz';
}
}
if ($bDestinationCompress)
{
$strDestinationFile .= '.gz';
}
# Run file copy in an eval block because some errors are expected
my $bReturn;
eval
{
($bReturn, $strCopyHash, $iCopySize) =
$oFile->copy($strSourcePathType, $strSourceFile,
$strDestinationPathType, $strDestinationFile,
$bSourceCompressed, $bDestinationCompress,
$bSourceIgnoreMissing, undef, '0770', false, undef, undef,
$bChecksumAppend);
return true;
}
# Check for errors after copy
or do
{
my $oException = $EVAL_ERROR;
if (isException($oException))
{
if ($bSourceMissing && !$bSourceIgnoreMissing)
{
next;
}
confess $oException;
}
confess $oException;
};
if ($bSourceMissing)
{
if ($bSourceIgnoreMissing)
{
if ($bReturn)
{
confess 'copy() returned ' . $bReturn . ' when ignore missing set';
}
next;
}
confess 'expected source file missing error';
}
if (!defined($strCopyHash))
{
confess 'copy hash must be defined';
}
if ($bChecksumAppend)
{
if ($bDestinationCompress)
{
$strDestinationFile =
substr($strDestinationFile, 0, length($strDestinationFile) -3) . "-${strSourceHash}.gz";
}
else
{
$strDestinationFile .= '-' . $strSourceHash;
}
}
unless (-e $strDestinationFile)
{
confess "could not find destination file ${strDestinationFile}";
}
my $strDestinationTest = $strDestinationFile;
if ($bDestinationCompress)
{
$strDestinationTest = substr($strDestinationFile, 0, length($strDestinationFile) - 3) . '.test';
system("gzip -dc ${strDestinationFile} > ${strDestinationTest}") == 0
or die "could not decompress ${strDestinationFile}";
}
my ($strDestinationHash, $iDestinationSize) = $oFile->hashSize(PATH_ABSOLUTE, $strDestinationTest);
if ($strSourceHash ne $strDestinationHash || $strSourceHash ne $strCopyHash)
{
confess
"source ${strSourceHash}, copy ${strCopyHash} and destination ${strDestinationHash} file hashes do not match";
}
if ($iSourceSize != $iDestinationSize || $iSourceSize != $iCopySize)
{
confess "source ${iSourceSize}, copy ${iCopySize} and destination ${iDestinationSize} sizes do not match";
}
}
}
}
}
}
}
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FileExistsTest.pm
-91
View File
@@ -1,91 +0,0 @@
####################################################################################################################################
# FileExistsTest.pm - Tests for File->exists()
####################################################################################################################################
package pgBackRestTest::Module::File::FileExistsTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote
foreach my $bRemote (false, true)
{
# Loop through exists
foreach my $bExists (false, true)
{
# Loop through exists
foreach my $bError ($bExists ? (false, true) : (false))
{
if (!$self->begin("rmt ${bRemote}, err ${bError}, exists ${bExists}")) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote, $bError);
my $strFile = $self->testPath() . '/test.txt';
if ($bError)
{
$strFile = $self->testPath() . '/private/test.txt';
}
elsif ($bExists)
{
executeTest("echo 'TESTDATA' > ${strFile}");
}
# Execute in eval in case of error
eval
{
if ($oFile->exists(PATH_BACKUP_ABSOLUTE, $strFile) != $bExists)
{
confess "bExists is set to ${bExists}, but exists() returned " . !$bExists;
}
return true;
}
or do
{
my $oException = $@;
my $iCode;
my $strMessage;
if (isException($oException))
{
$iCode = $oException->code();
$strMessage = $oException->message();
}
else
{
$strMessage = $oException;
}
if ($bError)
{
next;
}
confess 'error raised: ' . $strMessage . "\n";
};
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FileHashTest.pm
-100
View File
@@ -1,100 +0,0 @@
####################################################################################################################################
# FileHashTest.pm - Tests for File->hash()
####################################################################################################################################
package pgBackRestTest::Module::File::FileHashTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote
foreach my $bRemote (false, true)
{
# Loop through error
foreach my $bError (false, true)
{
# Loop through exists
foreach my $bExists (false, true)
{
# Loop through exists
foreach my $bCompressed (false, true)
{
if (!$self->begin("rmt ${bRemote}, err ${bError}, exists ${bExists}, cmp ${bCompressed}")) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote, $bError);
my $strFile = $self->testPath() . '/test.txt';
if ($bError)
{
$strFile = $self->testPath() . '/' . ($bRemote ? 'user' : 'backrest') . '_private/test.txt';
}
elsif (!$bExists)
{
$strFile = $self->testPath() . '/error.txt';
}
else
{
executeTest("echo 'TESTDATA' > ${strFile}");
if ($bCompressed && !$bRemote)
{
$oFile->compress(PATH_BACKUP_ABSOLUTE, $strFile);
$strFile = $strFile . '.gz';
}
}
# Execute in eval in case of error
my $strHash;
my $iSize;
my $bErrorExpected = !$bExists || $bError || $bRemote;
eval
{
($strHash, $iSize) = $oFile->hashSize(PATH_BACKUP_ABSOLUTE, $strFile, $bCompressed);
return true;
}
or do
{
if ($bErrorExpected)
{
next;
}
confess $EVAL_ERROR;
};
if ($bErrorExpected)
{
confess 'error was expected';
}
if ($strHash ne '06364afe79d801433188262478a76d19777ef351')
{
confess 'hashes do not match';
}
}
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FileListTest.pm
-124
View File
@@ -1,124 +0,0 @@
####################################################################################################################################
# FileListTest.pm - Tests for File->list()
####################################################################################################################################
package pgBackRestTest::Module::File::FileListTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote
foreach my $bRemote (false, true)
{
# Loop through sort
foreach my $strSort ('reverse', undef)
{
# Loop through expression
foreach my $strExpression (undef, "^test2\\..*\$", "^du\$")
{
# Loop through exists
foreach my $bExists (false, true)
{
# Loop through ignore missing
for (my $bIgnoreMissing = false; $bIgnoreMissing <= $bExists; $bIgnoreMissing++)
{
# Loop through error
foreach my $bError (false, true)
{
if (!$self->begin(
"rmt ${bRemote}, err ${bError}, exist ${bExists}, ignmis ${bIgnoreMissing}, " .
'exp ' . (defined($strExpression) ? $strExpression : '[undef]') . ', ' .
'srt ' . (defined($strSort) ? $strSort : '[undef]'))) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote, $bError);
my $strPath = $self->testPath();
if ($bError)
{
$strPath = $self->testPath() . '/' . ($bRemote ? 'user' : 'backrest') . '_private';
}
elsif (!$bExists)
{
$strPath = $self->testPath() . '/error';
}
else
{
executeTest("echo 'TESTDATA' > ${strPath}/test.txt");
executeTest("echo 'TESTDATA2' > ${strPath}/test2.txt");
}
my @stryFileCompare = split(/\n/, "test.txt\ntest2.txt");
# Execute in eval in case of error
my @stryFileList;
my $bErrorExpected = (!$bExists && !$bIgnoreMissing) || $bError;
eval
{
@stryFileList = $oFile->list(
PATH_BACKUP_ABSOLUTE, $strPath,
{strExpression => $strExpression, strSortOrder => $strSort, bIgnoreMissing => $bIgnoreMissing});
return true;
}
or do
{
if ($bErrorExpected)
{
next;
}
confess $EVAL_ERROR;
};
if ($bErrorExpected)
{
confess 'error was expected';
}
# Validate the list
if (defined($strExpression))
{
@stryFileCompare = grep(/$strExpression/i, @stryFileCompare);
}
if (defined($strSort))
{
@stryFileCompare = sort {$b cmp $a} @stryFileCompare;
}
my $strFileList = sprintf("@stryFileList");
my $strFileCompare = sprintf("@stryFileCompare");
if ($strFileList ne $strFileCompare)
{
confess "list (${strFileList})[" . @stryFileList .
"] does not match compare (${strFileCompare})[" . @stryFileCompare . ']';
}
}
}
}
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FileManifestTest.pm
-215
View File
@@ -1,215 +0,0 @@
####################################################################################################################################
# FileManifestTest.pm - Tests for File->manifest()
####################################################################################################################################
package pgBackRestTest::Module::File::FileManifestTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(basename dirname);
use IO::Socket::UNIX;
use pgBackRest::Common::Log;
use pgBackRest::Common::Exception;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
################################################################################################################################
if ($self->begin("FileCommon::fileManifestStat()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strFile = $self->testPath() . '/test.txt';
$self->testResult(sub {pgBackRest::FileCommon::fileManifestStat($strFile)}, '[undef]', 'ignore missing file');
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite($strFile, "TEST");
utime(1111111111, 1111111111, $strFile);
executeTest('chmod 1640 ' . $strFile);
$self->testResult(
sub {pgBackRest::FileCommon::fileManifestStat($strFile)},
'{group => ' . $self->group() .
', mode => 1640, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}',
'stat file');
#---------------------------------------------------------------------------------------------------------------------------
my $strSocketFile = $self->testPath() . '/test.socket';
# Create a socket to test invalid files
my $oSocket = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Local => $strSocketFile, Listen => 1);
$self->testException(
sub {pgBackRest::FileCommon::fileManifestStat($strSocketFile)}, ERROR_FILE_INVALID,
"${strSocketFile} is not of type directory, file, or link");
# Cleanup socket
$oSocket->close();
fileRemove($strSocketFile);
#---------------------------------------------------------------------------------------------------------------------------
my $strTestPath = $self->testPath() . '/public_dir';
filePathCreate($strTestPath, '0750');
$self->testResult(
sub {pgBackRest::FileCommon::fileManifestStat($strTestPath)},
'{group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}',
'stat directory');
#---------------------------------------------------------------------------------------------------------------------------
my $strTestLink = $self->testPath() . '/public_dir_link';
symlink($strTestPath, $strTestLink)
or confess &log(ERROR, "unable to create symlink from ${strTestPath} to ${strTestLink}");
$self->testResult(
sub {pgBackRest::FileCommon::fileManifestStat($strTestLink)},
'{group => ' . $self->group() . ", link_destination => ${strTestPath}, type => l, user => " . $self->pgUser() . '}',
'stat link');
}
################################################################################################################################
if ($self->begin("FileCommon::fileManifestList()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my @stryFile = ('.', 'test.txt');
$self->testResult(
sub {pgBackRest::FileCommon::fileManifestList($self->testPath(), \@stryFile)},
'{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}}',
'skip missing file');
}
################################################################################################################################
if ($self->begin("FileCommon::fileManifestRecurse()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strTestPath = $self->testPath() . '/public_dir';
my $strTestFile = "${strTestPath}/test.txt";
$self->testException(
sub {my $hManifest = {}; pgBackRest::FileCommon::fileManifestRecurse($strTestFile, undef, 0, $hManifest); $hManifest},
ERROR_FILE_MISSING, "unable to stat ${strTestFile}: No such file or directory");
#---------------------------------------------------------------------------------------------------------------------------
filePathCreate($strTestPath, '0750');
$self->testResult(
sub {my $hManifest = {}; pgBackRest::FileCommon::fileManifestRecurse($strTestPath, undef, 0, $hManifest); $hManifest},
'{. => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}}',
'empty directory manifest');
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite($strTestFile, "TEST");
utime(1111111111, 1111111111, $strTestFile);
executeTest('chmod 0750 ' . $strTestFile);
filePathCreate("${strTestPath}/sub", '0750');
$self->testResult(
sub {my $hManifest = {}; pgBackRest::FileCommon::fileManifestRecurse(
$self->testPath(), basename($strTestPath), 1, $hManifest); $hManifest},
'{public_dir => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'public_dir/sub => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'public_dir/' . basename($strTestFile) . ' => {group => ' . $self->group() .
', mode => 0750, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}}',
'directory and file manifest');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {my $hManifest = {}; pgBackRest::FileCommon::fileManifestRecurse($strTestFile, undef, 0, $hManifest); $hManifest},
'{' . basename($strTestFile) . ' => {group => ' . $self->group() .
', mode => 0750, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}}',
'single file manifest');
}
# Loop through local/remote
for (my $bRemote = false; $bRemote <= true; $bRemote++)
{
if (!$self->begin('File->manifest() => ' . ($bRemote ? 'remote' : 'local'))) {next}
# Create the file object
my $oFile = new pgBackRest::File
(
$self->stanza(),
$self->testPath(),
$bRemote ? $self->remote() : $self->local()
);
#---------------------------------------------------------------------------------------------------------------------------
my $strMissingFile = $self->testPath() . '/missing';
$self->testException(
sub {$oFile->manifest(PATH_BACKUP_ABSOLUTE, $strMissingFile)},
ERROR_FILE_MISSING, "unable to stat ${strMissingFile}: No such file or directory");
#---------------------------------------------------------------------------------------------------------------------------
# Setup test data
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1');
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1/sub2');
executeTest("echo 'TESTDATA' > " . $self->testPath() . '/test.txt');
utime(1111111111, 1111111111, $self->testPath() . '/test.txt');
executeTest('chmod 1640 ' . $self->testPath() . '/test.txt');
executeTest("echo 'TESTDATA_' > ". $self->testPath() . '/sub1/test-sub1.txt');
utime(1111111112, 1111111112, $self->testPath() . '/sub1/test-sub1.txt');
executeTest('chmod 0640 ' . $self->testPath() . '/sub1/test-sub1.txt');
executeTest("echo 'TESTDATA__' > " . $self->testPath() . '/sub1/sub2/test-sub2.txt');
utime(1111111113, 1111111113, $self->testPath() . '/sub1/sub2/test-sub2.txt');
executeTest('chmod 0646 ' . $self->testPath() . '/sub1/test-sub1.txt');
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/test-hardlink.txt');
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/sub2/test-hardlink.txt');
executeTest('ln -s .. ' . $self->testPath() . '/sub1/test');
executeTest('chmod 0700 ' . $self->testPath() . '/sub1/test');
executeTest('ln -s ../.. ' . $self->testPath() . '/sub1/sub2/test');
executeTest('chmod 0750 ' . $self->testPath() . '/sub1/sub2/test');
executeTest('chmod 0770 ' . $self->testPath());
$self->testResult(
sub {$oFile->manifest(PATH_BACKUP_ABSOLUTE, $self->testPath())},
'{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1/sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1/sub2/test => {group => ' . $self->group() . ', link_destination => ../.., type => l, user => ' .
$self->pgUser() . '}, ' .
'sub1/sub2/test-hardlink.txt => ' .
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/sub2/test-sub2.txt => ' .
'{group => ' . $self->group() . ', mode => 0666, modification_time => 1111111113, size => 11, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/test => {group => ' . $self->group() . ', link_destination => .., type => l, user => ' . $self->pgUser() . '}, ' .
'sub1/test-hardlink.txt => ' .
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/test-sub1.txt => ' .
'{group => ' . $self->group() . ', mode => 0646, modification_time => 1111111112, size => 10, type => f, user => ' .
$self->pgUser() . '}, ' .
'test.txt => ' .
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}}',
'complete manifest');
}
}
1;
test/lib/pgBackRestTest/Module/File/FileMoveTest.pm
-107
View File
@@ -1,107 +0,0 @@
####################################################################################################################################
# FileMoveTest.pm - Tests for File->move()
####################################################################################################################################
package pgBackRestTest::Module::File::FileMoveTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote ??? enable remote tests and have them throw an error
for (my $bRemote = 0; $bRemote <= 0; $bRemote++)
{
# Loop through source exists
for (my $bSourceExists = 0; $bSourceExists <= 1; $bSourceExists++)
{
# Loop through source errors
for (my $bSourceError = 0; $bSourceError <= 1; $bSourceError++)
{
# Loop through destination exists
for (my $bDestinationExists = 0; $bDestinationExists <= 1; $bDestinationExists++)
{
# Loop through source errors
for (my $bDestinationError = 0; $bDestinationError <= 1; $bDestinationError++)
{
# Loop through create
for (my $bCreate = 0; $bCreate <= $bDestinationExists; $bCreate++)
{
# Increment the run, log, and decide whether this unit test should be run
if (!$self->begin(
"src_exists ${bSourceExists}, src_error ${bSourceError}, " .
", dst_exists ${bDestinationExists}, dst_error ${bDestinationError}, dst_create ${bCreate}")) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote, $bSourceError || $bDestinationError);
my $strSourceFile = $self->testPath() . '/test.txt';
my $strDestinationFile = $self->testPath() . '/test-dest.txt';
if ($bSourceError)
{
$strSourceFile = $self->testPath() . '/' . ($bRemote ? 'user' : 'backrest') . "_private/test.txt";
}
elsif ($bSourceExists)
{
executeTest("echo 'TESTDATA' > ${strSourceFile}");
}
if ($bDestinationError)
{
$strDestinationFile = $self->testPath() . '/' . ($bRemote ? 'user' : 'backrest') . "_private/test.txt";
}
elsif (!$bDestinationExists)
{
$strDestinationFile = $self->testPath() . '/sub/test-dest.txt';
}
# Execute in eval in case of error
eval
{
$oFile->move(PATH_BACKUP_ABSOLUTE, $strSourceFile, PATH_BACKUP_ABSOLUTE, $strDestinationFile, $bCreate);
return true;
}
or do
{
if (!$bSourceExists || (!$bDestinationExists && !$bCreate) || $bSourceError || $bDestinationError)
{
next;
}
confess $EVAL_ERROR;
};
if (!$bSourceExists || (!$bDestinationExists && !$bCreate) || $bSourceError || $bDestinationError)
{
confess 'error should have been raised';
}
unless (-e $strDestinationFile)
{
confess 'file was not moved';
}
}
}
}
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FileOwnerTest.pm
-105
View File
@@ -1,105 +0,0 @@
####################################################################################################################################
# FileOwnerTest.pm - Tests for File->owner()
####################################################################################################################################
package pgBackRestTest::Module::File::FileOwnerTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote
foreach my $bRemote (false, true)
{
# Loop through exists
foreach my $bExists (!$bRemote ? (false, true) : (false))
{
# Loop through exists
foreach my $bError ($bExists && !$bRemote ? (false, true) : (false))
{
# Loop through users
foreach my $strUser ($bExists && !$bError && !$bRemote ? ($self->pgUser(), BOGUS, undef) : ($self->pgUser()))
{
# Loop through group
foreach my $strGroup (
$bExists && !$bError && !$bRemote && defined($strUser) && $strUser ne BOGUS ? ($self->group(), BOGUS, undef) : ($self->group()))
{
if (!$self->begin(
"rmt ${bRemote}, err ${bError}, exists ${bExists}, user " . (defined($strUser) ? $strUser : '[undef]') .
", group " . (defined($strGroup) ? $strGroup : '[undef]'))) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote, $bError);
# Create the test file
my $strFile = $self->testPath() . '/test.txt';
if ($bExists)
{
executeTest("echo 'TESTDATA' > ${strFile}");
if ($bError)
{
executeTest("sudo chown root:root ${strFile}");
}
}
my $strFunction = sub {$oFile->owner(PATH_BACKUP_ABSOLUTE, $strFile, $strUser, $strGroup)};
# Remote operation not allowed
if ($bRemote)
{
$self->testException($strFunction, ERROR_ASSERT, "pgBackRest::File->owner: remote operation not supported");
}
# Error on not exists
elsif (!$bExists)
{
$self->testException($strFunction, ERROR_FILE_MISSING, "${strFile} does not exist");
}
# Else permission error
elsif ($bError)
{
$self->testException(
$strFunction, ERROR_FILE_OWNER, "unable to set ownership for '${strFile}': Operation not permitted");
}
# Else bogus user
elsif (defined($strUser) && $strUser eq BOGUS)
{
$self->testException($strFunction, ERROR_USER_MISSING, "user '" . BOGUS . "' does not exist");
}
# Else bogus group
elsif (defined($strGroup) && $strGroup eq BOGUS)
{
$self->testException($strFunction, ERROR_GROUP_MISSING, "group '" . BOGUS . "' does not exist");
}
# Else success
else
{
$self->testResult($strFunction, '[undef]', 'success');
}
}
}
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FilePathCreateTest.pm
-115
View File
@@ -1,115 +0,0 @@
####################################################################################################################################
# FilePathCreateTest.pm - Tests for File->pathCreate()
####################################################################################################################################
package pgBackRestTest::Module::File::FilePathCreateTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Fcntl qw(:mode);
use File::stat;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote
for (my $bRemote = 0; $bRemote <= 1; $bRemote++)
{
# Loop through error
for (my $bError = 0; $bError <= 1; $bError++)
{
# Loop through mode (mode will be set on true)
foreach my $strMode (undef, '0700')
{
my $strPathType = PATH_BACKUP_CLUSTER;
# Increment the run, log, and decide whether this unit test should be run
if (!$self->begin("rmt ${bRemote}, err ${bError}, mode " . (defined($strMode) ? $strMode : 'undef'))) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote, $bError);
filePathCreate($self->testPath() . '/backup', '770');
filePathCreate($self->testPath() . '/backup/db', '770');
my $strPath = 'path';
# If not exists then set the path to something bogus
if ($bError)
{
$strPath = $self->testPath() . '/' . ($bRemote ? 'user' : 'backrest') . "_private/path";
$strPathType = PATH_BACKUP_ABSOLUTE;
}
# Execute in eval to catch errors
my $bErrorExpected = $bError;
eval
{
$oFile->pathCreate($strPathType, $strPath, $strMode);
return true;
}
# Check for errors
or do
{
# Ignore errors if the path did not exist
if ($bErrorExpected)
{
next;
}
confess $EVAL_ERROR;
};
if ($bErrorExpected)
{
confess 'error was expected';
}
# Make sure the path was actually created
my $strPathCheck = $oFile->pathGet($strPathType, $strPath);
unless (-e $strPathCheck)
{
confess 'path was not created';
}
# Check that the mode was set correctly
my $oStat = lstat($strPathCheck);
if (!defined($oStat))
{
confess "unable to stat ${strPathCheck}";
}
if (defined($strMode))
{
if ($strMode ne sprintf('%04o', S_IMODE($oStat->mode)))
{
confess "mode was not set to {$strMode}";
}
}
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FileRemoveTest.pm
-112
View File
@@ -1,112 +0,0 @@
####################################################################################################################################
# FileRemoveTest.pm - Tests for File->remove()
####################################################################################################################################
package pgBackRestTest::Module::File::FileRemoveTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote
foreach my $bRemote (false, true)
{
# Loop through exists
foreach my $bError (false, true)
{
# Loop through exists
foreach my $bExists (false, true)
{
# Loop through temp
foreach my $bTemp (false, true)
{
# Loop through ignore missing
foreach my $bIgnoreMissing (false, true)
{
if (!$self->begin(
"rmt ${bRemote}, err = ${bError}, exists ${bExists}, tmp ${bTemp}, ignmis ${bIgnoreMissing}")) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote, $bError);
my $strFile = $self->testPath() . '/test.txt';
if ($bError)
{
$strFile = $self->testPath() . '/' . ($bRemote ? 'user' : 'backrest') . '_private/test.txt';
}
elsif (!$bExists)
{
$strFile = $self->testPath() . '/private/error.txt';
}
else
{
executeTest("echo 'TESTDATA' > ${strFile}" . ($bTemp ? '.pgbackrest.tmp' : ''));
}
# Execute in eval in case of error
my $bRemoved;
eval
{
$bRemoved = $oFile->remove(PATH_BACKUP_ABSOLUTE, $strFile, $bTemp, $bIgnoreMissing);
return true;
}
or do
{
if ($bError || $bRemote)
{
next;
}
if (!$bExists && !$bIgnoreMissing)
{
next;
}
confess $EVAL_ERROR;
};
if ($bError || $bRemote)
{
confess 'error should have been returned';
}
if (!$bRemoved)
{
if (!$bExists && $bIgnoreMissing)
{
next;
}
confess 'remove returned false, but something should have been removed';
}
if (-e ($strFile . ($bTemp ? '.pgbackrest.tmp' : '')))
{
confess 'file still exists';
}
}
}
}
}
}
}
1;
test/lib/pgBackRestTest/Module/File/FileStatTest.pm
-64
View File
@@ -1,64 +0,0 @@
####################################################################################################################################
# FileStatTest.pm - Tests for FileCommon::fileStat
####################################################################################################################################
package pgBackRestTest::Module::File::FileStatTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Fcntl qw(:mode);
use File::stat;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::FileCommon;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
################################################################################################################################
if ($self->begin("FileCommon::fileStat()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strFile = $self->testPath() . '/test.txt';
$self->testException(sub {fileStat($strFile)}, ERROR_FILE_MISSING, "unable to stat ${strFile}: No such file or directory");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {fileStat($strFile, true)}, '[undef]', 'ignore missing file');
#---------------------------------------------------------------------------------------------------------------------------
fileStringWrite($strFile);
$self->testResult(sub {fileStat($strFile)}, '[object]', 'stat file');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {S_ISREG(fileStat($strFile)->mode)}, true, 'check stat file result');
#---------------------------------------------------------------------------------------------------------------------------
my $strPrivateDir = $self->testPath() . '/private_dir';
my $strPrivateFile = "${strPrivateDir}/test.txt";
executeTest("sudo mkdir -m 700 ${strPrivateDir}");
$self->testException(
sub {fileStat($strPrivateFile, true)}, ERROR_FILE_OPEN, "unable to stat ${strPrivateFile}: Permission denied");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {S_ISDIR(fileStat($strPrivateDir)->mode)}, true, 'check stat directory result');
}
}
1;
test/lib/pgBackRestTest/Module/File/FileUnitTest.pm
-137
View File
@@ -1,137 +0,0 @@
####################################################################################################################################
# FileUnitTest.pm - Unit tests for File module.
####################################################################################################################################
package pgBackRestTest::Module::File::FileUnitTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::File;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Increment the run, log, and decide whether this unit test should be run
if (!$self->begin('unit')) {return}
# Setup test directory and get file object
my $oLocalFile = $self->setup(false, false);
# Test File->pathTypeGet()
#---------------------------------------------------------------------------------------------------------------------------
{
&log(INFO, " File->pathTypeGet()");
$self->testResult(sub {$oLocalFile->pathTypeGet(PATH_ABSOLUTE)}, PATH_ABSOLUTE, 'absolute path type');
$self->testResult(sub {$oLocalFile->pathTypeGet(PATH_DB)}, PATH_DB, 'db path type');
$self->testResult(sub {$oLocalFile->pathTypeGet(PATH_DB_ABSOLUTE)}, PATH_DB, 'db absolute path type');
$self->testResult(sub {$oLocalFile->pathTypeGet(PATH_BACKUP)}, PATH_BACKUP, 'backup path type');
$self->testResult(sub {$oLocalFile->pathTypeGet(PATH_BACKUP_ARCHIVE)}, PATH_BACKUP, 'backup archive path type');
$self->testException(sub {$oLocalFile->pathTypeGet('bogus')}, ERROR_ASSERT, "no known path types in 'bogus'");
}
# Test File->pathGet()
#---------------------------------------------------------------------------------------------------------------------------
{
&log(INFO, " File->pathGet()");
# Test temp file errors
$self->testException(
sub {$oLocalFile->pathGet(PATH_BACKUP, 'test', true)},
ERROR_ASSERT, "temp file not supported for path type 'backup'");
$self->testException(
sub {$oLocalFile->pathGet(PATH_ABSOLUTE, undef, true)},
ERROR_ASSERT, "strFile must be defined when temp file specified");
$self->testException(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE, undef, true)},
ERROR_ASSERT, "strFile must be defined when temp file specified");
$self->testException(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE_OUT, undef, true)},
ERROR_ASSERT, "strFile must be defined when temp file specified");
$self->testException(
sub {$oLocalFile->pathGet(PATH_BACKUP_TMP, undef, true)},
ERROR_ASSERT, "strFile must be defined when temp file specified");
# Test absolute path
$self->testException(
sub {$oLocalFile->pathGet(PATH_ABSOLUTE)}, ERROR_ASSERT, "strFile must be defined for absolute path");
$self->testException(
sub {$oLocalFile->pathGet(PATH_ABSOLUTE, 'file')}, ERROR_ASSERT, "absolute path absolute:file must start with /");
$self->testResult(sub {$oLocalFile->pathGet(PATH_ABSOLUTE, '/file', true)}, "/file.pgbackrest.tmp", 'absolute path temp');
$self->testResult(sub {$oLocalFile->pathGet(PATH_ABSOLUTE, '/file')}, "/file", 'absolute path file');
# Test backup path
$self->testResult(sub {$oLocalFile->pathGet(PATH_BACKUP, 'file')}, $self->testPath() . '/file', 'backup path file');
$self->testResult(sub {$oLocalFile->pathGet(PATH_BACKUP, undef)}, $self->testPath(), 'backup path');
# Error when stanza not defined
$self->testException(
sub {(new pgBackRest::File(undef, $self->testPath(), $self->local()))->pathGet(PATH_BACKUP_TMP)},
ERROR_ASSERT, "strStanza not defined");
# Test backup tmp path
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_TMP, 'file', true)}, $self->testPath() . '/temp/db.tmp/file.pgbackrest.tmp',
'backup temp path temp file');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_TMP, 'file')}, $self->testPath() . '/temp/db.tmp/file', 'backup temp path file');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_TMP, undef)}, $self->testPath() . '/temp/db.tmp', 'backup temp path');
# Test archive path
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE, undef)}, $self->testPath() . '/archive/db', 'archive path');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE, '9.3-1')}, $self->testPath() . '/archive/db/9.3-1', 'archive id path');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE, '9.3-1/000000010000000100000001')},
$self->testPath() . '/archive/db/9.3-1/0000000100000001/000000010000000100000001',
'archive path file');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE, '9.3-1/000000010000000100000001', true)},
$self->testPath() . '/archive/db/9.3-1/0000000100000001/000000010000000100000001.pgbackrest.tmp',
'archive path temp file');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE, '9.3-1/00000001.history')},
$self->testPath() . '/archive/db/9.3-1/00000001.history',
'archive path history file');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE, '9.3-1/00000001.history', true)},
$self->testPath() . '/archive/db/9.3-1/00000001.history.pgbackrest.tmp',
'archive path history temp file');
# Test archive out path
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE_OUT, '000000010000000100000001')},
$self->testPath() . '/archive/db/out/000000010000000100000001',
'archive out path file');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE_OUT)}, $self->testPath() . '/archive/db/out', 'archive out path');
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_ARCHIVE_OUT, '000000010000000100000001', true)},
$self->testPath() . '/archive/db/out/000000010000000100000001.pgbackrest.tmp',
'archive out path temp file');
# Test backup cluster path
$self->testResult(
sub {$oLocalFile->pathGet(PATH_BACKUP_CLUSTER, 'file')}, $self->testPath() . '/backup/db/file', 'cluster path file');
$self->testResult(sub {$oLocalFile->pathGet(PATH_BACKUP_CLUSTER)}, $self->testPath() . '/backup/db', 'cluster path');
# Test invalid path type
$self->testException(sub {$oLocalFile->pathGet('bogus')}, ERROR_ASSERT, "no known path types in 'bogus'");
}
}
1;
test/lib/pgBackRestTest/Module/File/FileWaitTest.pm
-71
View File
@@ -1,71 +0,0 @@
####################################################################################################################################
# FileWaitTest.pm - Tests for File->wait()
####################################################################################################################################
package pgBackRestTest::Module::File::FileWaitTest;
use parent 'pgBackRestTest::Module::File::FileCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use POSIX qw(ceil);
use Time::HiRes qw(gettimeofday usleep);
use pgBackRest::Common::Log;
use pgBackRest::File;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Loop through local/remote
for (my $bRemote = 0; $bRemote <= 1; $bRemote++)
{
my $lTimeBegin = gettimeofday();
if (!$self->begin("rmt ${bRemote}, begin ${lTimeBegin}")) {next}
# Setup test directory and get file object
my $oFile = $self->setup($bRemote);
# If there is not enough time to complete the test then sleep
if (ceil($lTimeBegin) - $lTimeBegin < .250)
{
my $lSleepMs = ceil(((int($lTimeBegin) + 1) - $lTimeBegin) * 1000);
usleep($lSleepMs * 1000);
&log(DEBUG, "slept ${lSleepMs}ms: begin ${lTimeBegin}, end " . gettimeofday());
$lTimeBegin = gettimeofday();
}
# Run the test
my $lTimeBeginCheck = $oFile->wait(PATH_DB_ABSOLUTE);
&log(DEBUG, "begin ${lTimeBegin}, check ${lTimeBeginCheck}, end " . time());
# Current time should have advanced by 1 second
if (int(time()) == int($lTimeBegin))
{
confess "time was not advanced by 1 second";
}
# lTimeBegin and lTimeBeginCheck should be equal
if (int($lTimeBegin) != $lTimeBeginCheck)
{
confess 'time begin ' || int($lTimeBegin) || "and check ${lTimeBeginCheck} should be equal";
}
}
}
1;
test/lib/pgBackRestTest/Module/Full/FullRealTest.pm
+40 -33
View File
@@ -22,10 +22,9 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
@@ -67,7 +66,7 @@ sub run
}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
false, $self->expect(),
{bHostBackup => $bHostBackup, bStandby => $bHostStandby, strBackupDestination => $strBackupDestination,
bCompress => $bCompress, bArchiveAsync => $bArchiveAsync});
@@ -116,7 +115,8 @@ sub run
$strType = BACKUP_TYPE_FULL;
# Remove the files in the archive directory
executeTest('sudo rm -rf ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE) . "/*");
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE, {bRecurse => true});
$oHostDbMaster->check(
'fail on missing archive.info file',
{iTimeout => 0.1, iExpectedExitStatus => ERROR_FILE_MISSING});
@@ -177,7 +177,7 @@ sub run
# load the archive info file and munge it for testing by breaking the database version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
$oHostDbMaster->check($strComment, {iTimeout => 0.1, iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH});
@@ -189,7 +189,7 @@ sub run
}
# Restore the file to its original condition
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
# Check archive_timeout error when WAL segment is not found
$strComment = 'fail on archive timeout';
@@ -212,7 +212,7 @@ sub run
# Load the backup.info file and munge it for testing by breaking the database version and system id
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB =>
{&INFO_BACKUP_KEY_DB_VERSION => '8.0', &INFO_BACKUP_KEY_SYSTEM_ID => 6999999999999999999}});
@@ -226,7 +226,7 @@ sub run
}
# Restore the file to its original condition
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO));
# Providing a sufficient archive-timeout, verify that the check command runs successfully now with valid
# archive.info and backup.info files
@@ -245,8 +245,12 @@ sub run
# Stanza Create
#-----------------------------------------------------------------------------------------------------------------------
# With data existing in the archive and backup directory, remove backup info file and confirm failure
executeTest('sudo rm ' . $oHostBackup->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
# With data existing in the archive and backup directory, remove info files and confirm failure
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO);
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT);
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE);
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT);
$oHostBackup->stanzaCreate('fail on backup info file missing from non-empty dir',
{iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
@@ -254,20 +258,24 @@ sub run
$oHostBackup->stanzaCreate('verify success with force', {strOptionalParam => ' --' . OPTION_FORCE});
# Remove the backup info file
executeTest('sudo rm ' . $oHostBackup->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO);
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT);
# Change the database version by copying a new pg_control file to a new db-path to use for db mismatch test
filePathCreate($oHostDbMaster->dbPath() . '/testbase/' . DB_PATH_GLOBAL, undef, true, true);
storageDb()->pathCreate(
$oHostDbMaster->dbPath() . '/testbase/' . DB_PATH_GLOBAL,
{strMode => '0700', bIgnoreExists => true, bCreateParent => true});
if ($self->pgVersion() eq PG_VERSION_94)
{
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_95 . '.bin ' . $oHostDbMaster->dbPath() .
'/testbase/' . DB_FILE_PGCONTROL);
storageDb()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_95 . '.bin',
$oHostDbMaster->dbPath() . '/testbase/' . DB_FILE_PGCONTROL);
} else
{
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbPath() .
'/testbase/' . DB_FILE_PGCONTROL);
storageDb()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin',
$oHostDbMaster->dbPath() . '/testbase/' . DB_FILE_PGCONTROL);
}
# Run stanza-create online to confirm proper handling of configValidation error against new db-path
@@ -281,8 +289,8 @@ sub run
$oHostBackup->stanzaCreate('successfully create stanza files to be upgraded',
{strOptionalParam => ' --' . OPTION_DB_PATH . '=' . $oHostDbMaster->dbPath() . '/testbase/ --no-' . OPTION_ONLINE .
' --' . OPTION_FORCE});
my $oAchiveInfo = new pgBackRest::Archive::ArchiveInfo($oHostBackup->repoPath() . '/archive/' . $self->stanza());
my $oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoPath() . '/backup/' . $self->stanza());
my $oAchiveInfo = new pgBackRest::Archive::ArchiveInfo(storageRepo()->pathGet('archive/' . $self->stanza()));
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
# Read info files to confirm the files were created with a different database version
if ($self->pgVersion() eq PG_VERSION_94)
@@ -304,8 +312,8 @@ sub run
$oHostBackup->stanzaUpgrade('upgrade stanza files online');
# Reread the info files and confirm the result
$oAchiveInfo = new pgBackRest::Archive::ArchiveInfo($oHostBackup->repoPath() . '/archive/' . $self->stanza());
$oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoPath() . '/backup/' . $self->stanza());
$oAchiveInfo = new pgBackRest::Archive::ArchiveInfo(storageRepo()->pathGet('archive/' . $self->stanza()));
$oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
$self->testResult(sub {$oAchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
$self->pgVersion())}, true, 'archive upgrade online corrects db');
$self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef,
@@ -470,12 +478,12 @@ sub run
$strType = BACKUP_TYPE_INCR;
# Create a tablespace directory
filePathCreate($oHostDbMaster->tablespacePath(1), undef, undef, true);
storageTest()->pathCreate($oHostDbMaster->tablespacePath(1), {strMode => '0700', bCreateParent => true});
# Also create it on the standby so replay won't fail
if (defined($oHostDbStandby))
{
filePathCreate($oHostDbStandby->tablespacePath(1), undef, undef, true);
storageTest()->pathCreate($oHostDbStandby->tablespacePath(1), {strMode => '0700', bCreateParent => true});
}
$oHostDbMaster->sqlExecute(
@@ -508,7 +516,7 @@ sub run
$strType, 'update during backup',
{strTest => TEST_MANIFEST_BUILD, fTestDelay => $fTestDelay,
strOptionalParam => '--' . OPTION_STOP_AUTO . ' --no-' . OPTION_BACKUP_ARCHIVE_CHECK .
' --' . OPTION_BUFFER_SIZE . '=24576'});
' --' . OPTION_BUFFER_SIZE . '=32768'});
# Drop a table
$oHostDbMaster->sqlExecute('drop table test_remove');
@@ -613,11 +621,11 @@ sub run
# Drop and recreate db path
testPathRemove($oHostDbMaster->dbBasePath());
filePathCreate($oHostDbMaster->dbBasePath());
storageTest()->pathCreate($oHostDbMaster->dbBasePath(), {strMode => '0700'});
testPathRemove($oHostDbMaster->dbPath() . '/pg_xlog');
filePathCreate($oHostDbMaster->dbPath() . '/pg_xlog');
storageTest()->pathCreate($oHostDbMaster->dbPath() . '/pg_xlog', {strMode => '0700'});
testPathRemove($oHostDbMaster->tablespacePath(1));
filePathCreate($oHostDbMaster->tablespacePath(1));
storageTest()->pathCreate($oHostDbMaster->tablespacePath(1), {strMode => '0700'});
# Now the restore should work
$strComment = undef;
@@ -703,8 +711,8 @@ sub run
'--tablespace-map-all=../../tablespace', false);
# Save recovery file to test so we can use it in the next test
$oFile->copy(PATH_ABSOLUTE, $oHostDbMaster->dbBasePath() . '/recovery.conf',
PATH_ABSOLUTE, $self->testPath() . '/recovery.conf');
storageDb()->copy(
$oHostDbMaster->dbBasePath() . qw{/} . DB_FILE_RECOVERYCONF, $self->testPath() . qw{/} . DB_FILE_RECOVERYCONF);
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $strXidMessage);
@@ -736,8 +744,7 @@ sub run
executeTest('rm -rf ' . $oHostDbMaster->tablespacePath(1) . "/*");
# Restore recovery file that was saved in last test
$oFile->move(PATH_ABSOLUTE, $self->testPath . '/recovery.conf',
PATH_ABSOLUTE, $oHostDbMaster->dbBasePath() . '/recovery.conf');
storageDb()->move($self->testPath . '/recovery.conf', $oHostDbMaster->dbBasePath() . '/recovery.conf');
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, undef, undef, $bDelta, $bForce, $strType, $strTarget, $bTargetExclusive,
@@ -869,7 +876,7 @@ sub run
if ($bTestExtra)
{
# Create a postmaster.pid file so it appears that the server is running
fileStringWrite($oHostDbMaster->dbBasePath() . '/postmaster.pid', '99999');
storageTest()->put($oHostDbMaster->dbBasePath() . '/postmaster.pid', '99999');
# Incr backup - make sure a --no-online backup fails
#-----------------------------------------------------------------------------------------------------------------------
test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm
+66 -60
View File
@@ -22,11 +22,10 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
use pgBackRest::LibC qw(:checksum);
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
@@ -69,7 +68,7 @@ sub run
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, hardlink ${bHardLink}", $self->processMax() == 1)) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bHardLink => $bHardLink});
# Determine if this is a neutral test, i.e. we only want to do it once for local and once for remote. Neutral means
@@ -108,7 +107,7 @@ sub run
'184473f470864e067ee3a22e64b47b0a1c356f29', $lTime, undef, true);
# Load sample page
my $tBasePage = fileStringRead($self->dataPath() . '/page.bin');
my $tBasePage = ${storageTest()->get($self->dataPath() . '/page.bin')};
my $iBasePageChecksum = 0x1B99;
# Create base path
@@ -272,7 +271,7 @@ sub run
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . OPTION_ONLINE});
# Create a file link
filePathCreate($oHostDbMaster->dbPath() . '/pg_config', undef, undef, true);
storageTest()->pathCreate($oHostDbMaster->dbPath() . '/pg_config', {strMode => '0700', bCreateParent => true});
testFileCreate(
$oHostDbMaster->dbPath() . '/pg_config/postgresql.conf', "listen_addresses = *\n", $lTime - 100);
testLinkCreate($oHostDbMaster->dbPath() . '/pg_config/postgresql.conf.link', './postgresql.conf');
@@ -306,7 +305,7 @@ sub run
$oHostDbMaster->manifestLinkRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.conf.bad');
# Create stat directory link and file
filePathCreate($oHostDbMaster->dbPath() . '/pg_stat', undef, undef, true);
storageTest()->pathCreate($oHostDbMaster->dbPath() . '/pg_stat', {strMode => '0700', bCreateParent => true});
$oHostDbMaster->manifestLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_stat', '../pg_stat');
$oHostDbMaster->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA . '/pg_stat', 'global.stat', 'stats',
'e350d5ce0153f3e22d5db21cf2a4eff00f3ee877', $lTime - 100, undef, true);
@@ -315,10 +314,9 @@ sub run
$strFullBackup = $oHostBackup->backup(
$strType, 'create pg_stat link, pg_clog dir',
{oExpectedManifest => \%oManifest,
strOptionalParam => $strOptionalParam . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') .
' --no-' . OPTION_REPO_SYNC . ' --' . OPTION_BUFFER_SIZE . '=16384 --' . OPTION_CHECKSUM_PAGE,
strTest => $strTestPoint,
fTestDelay => 0});
strOptionalParam => $strOptionalParam . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . ' --' . OPTION_BUFFER_SIZE .
'=16384 --' . OPTION_CHECKSUM_PAGE,
strRepoType => REPO_TYPE_CIFS, strTest => $strTestPoint, fTestDelay => 0});
# Error on backup option to check logging
#-----------------------------------------------------------------------------------------------------------------------
@@ -336,10 +334,7 @@ sub run
$oHostBackup->backup(
$strType, 'protocol timeout',
{oExpectedManifest => \%oManifest, strOptionalParam => '--protocol-timeout=1 --db-timeout=.1',
strTest => TEST_BACKUP_START, fTestDelay => 1, iExpectedExitStatus => ERROR_PROTOCOL_TIMEOUT});
# Remove the aborted backup so the next backup is not a resume
testPathRemove($oHostBackup->repoPath() . '/temp/' . $self->stanza() . '.tmp');
strTest => TEST_BACKUP_START, fTestDelay => 1, iExpectedExitStatus => ERROR_FILE_READ});
}
# Stop operations and make sure the correct error occurs
@@ -397,10 +392,6 @@ sub run
}
}
# Cleanup any garbage left in the temp backup path
executeTest(
'sudo rm -rf ' . $oHostBackup->repoPath() . '/temp/' . $self->stanza() . '.tmp', {bRemote => $bRemote});
# Resume Full Backup
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_FULL;
@@ -414,40 +405,47 @@ sub run
# Create files in root tblspc paths that should not be copied or deleted.
# This will be checked later after a --force restore.
my $strDoNotDeleteFile = $oHostDbMaster->tablespacePath(1, 2) . '/donotdelete.txt';
filePathCreate(dirname($strDoNotDeleteFile), undef, undef, true);
storageTest()->pathCreate(dirname($strDoNotDeleteFile), {strMode => '0700', bCreateParent => true});
testFileCreate($strDoNotDeleteFile, 'DONOTDELETE-1-2');
filePathCreate($oHostDbMaster->tablespacePath(1), undef, undef, true);
storageTest()->pathCreate($oHostDbMaster->tablespacePath(1), {strMode => '0700', bCreateParent => true});
testFileCreate($oHostDbMaster->tablespacePath(1) . '/donotdelete.txt', 'DONOTDELETE-1');
filePathCreate($oHostDbMaster->tablespacePath(2), undef, undef, true);
storageTest()->pathCreate($oHostDbMaster->tablespacePath(2), {strMode => '0700', bCreateParent => true});
testFileCreate($oHostDbMaster->tablespacePath(2) . '/donotdelete.txt', 'DONOTDELETE-2');
filePathCreate($oHostDbMaster->tablespacePath(2, 2), undef, undef, true);
storageTest()->pathCreate($oHostDbMaster->tablespacePath(2, 2), {strMode => '0700', bCreateParent => true});
testFileCreate($oHostDbMaster->tablespacePath(2, 2) . '/donotdelete.txt', 'DONOTDELETE-2-2');
filePathCreate($oHostDbMaster->tablespacePath(11), undef, undef, true);
storageTest()->pathCreate($oHostDbMaster->tablespacePath(11), {strMode => '0700', bCreateParent => true});
my $strTmpPath = $oHostBackup->repoPath() . '/temp/' . $self->stanza() . '.tmp';
executeTest("sudo chmod g+w " . dirname($strTmpPath));
# Resume by copying the valid full backup over the last aborted full backup if it exists, or by creating a new path
my $strResumeBackup = (storageRepo()->list(
STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))[0];
my $strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . '/' .
($strResumeBackup ne $strFullBackup ? $strResumeBackup : backupLabel(storageRepo(), $strType, undef, time())));
testPathMove($oHostBackup->repoPath() . '/backup/' . $self->stanza() . "/${strFullBackup}", $strTmpPath);
forceStorageRemove(storageRepo(), $strResumePath, {bRecurse => true});
forceStorageMove(storageRepo(), 'backup/' . $self->stanza() . "/${strFullBackup}", $strResumePath);
$oHostBackup->infoMunge(
"$strTmpPath/" . FILE_MANIFEST,
$oHostBackup->manifestMunge(
basename($strResumePath),
{&MANIFEST_SECTION_TARGET_FILE =>
{(&MANIFEST_TARGET_PGDATA . '/' . &DB_FILE_PGVERSION) => {&MANIFEST_SUBKEY_CHECKSUM => undef}}},
false);
# Remove the main manifest so the backup appears aborted
forceStorageRemove(storageRepo(), "${strResumePath}/" . FILE_MANIFEST);
# Create a temp file in backup temp root to be sure it's deleted correctly
executeTest("touch ${strTmpPath}/file.tmp" . ($bCompress ? '.gz' : ''),
executeTest("sudo touch ${strResumePath}/file.tmp" . ($bCompress ? '.gz' : ''),
{bRemote => $bRemote});
executeTest("sudo chown -R " . $oHostBackup->userGet() . ' ' . dirname($strTmpPath));
executeTest("sudo chown " . BACKREST_USER . " ${strResumePath}/file.tmp" . ($bCompress ? '.gz' : ''));
$strFullBackup = $oHostBackup->backup(
$strType, 'resume',
{oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_RESUME,
strOptionalParam => '--force --' . OPTION_CHECKSUM_PAGE});
# Remove postmaster.pid so restore will succeed (the rest will be cleaned up)
testFileRemove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
# Remove postmaster.pid so restore will succeed (the rest will be cleaned up by the delta)
storageDb->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
# Misconfigure repo-path and check errors
#-----------------------------------------------------------------------------------------------------------------------
@@ -599,7 +597,7 @@ sub run
# Break the database version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_DB_VERSION => '8.0'}});
$oHostBackup->backup(
@@ -609,7 +607,7 @@ sub run
# Break the database system id
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 6999999999999999999}});
$oHostBackup->backup(
@@ -619,7 +617,7 @@ sub run
# Break the control version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_CONTROL => 842}});
$oHostBackup->backup(
@@ -629,7 +627,7 @@ sub run
# Break the catalog version
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO),
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_CATALOG => 197208141}});
$oHostBackup->backup(
@@ -638,7 +636,7 @@ sub run
strOptionalParam => '--log-level-console=detail'});
# Restore the file to its original condition
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO));
# Test broken tablespace configuration
#-----------------------------------------------------------------------------------------------------------------------
@@ -646,7 +644,7 @@ sub run
my $strTblSpcPath = $oHostDbMaster->dbBasePath() . '/' . DB_PATH_PGTBLSPC;
# Create a directory in pg_tablespace
filePathCreate("${strTblSpcPath}/path");
storageTest()->pathCreate("${strTblSpcPath}/path", {strMode => '0700', bCreateParent => true});
$oHostBackup->backup(
$strType, 'invalid path in ' . DB_PATH_PGTBLSPC,
@@ -769,18 +767,23 @@ sub run
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_INCR;
# Move database from backup to temp
$strTmpPath = $oHostBackup->repoPath() . '/temp/' .$self->stanza() . '.tmp';
# Create resumable backup from last backup
$strResumePath =
storageRepo()->pathGet('backup/' . $self->stanza() . '/' .
backupLabel(storageRepo(), $strType, substr($strBackup, 0, 16), time()));
testPathMove($oHostBackup->repoPath() . '/backup/' . $self->stanza() . "/${strBackup}", $strTmpPath);
executeTest("sudo chown -R " . $oHostBackup->userGet() . ' ' . dirname($strTmpPath));
forceStorageRemove(storageRepo(), $strResumePath);
forceStorageMove(storageRepo(), 'backup/' . $self->stanza() . "/${strBackup}", $strResumePath);
$oHostBackup->infoMunge(
"$strTmpPath/" . FILE_MANIFEST,
$oHostBackup->manifestMunge(
basename($strResumePath),
{&MANIFEST_SECTION_TARGET_FILE =>
{(&MANIFEST_TARGET_PGDATA . '/badchecksum.txt') => {&MANIFEST_SUBKEY_CHECKSUM => BOGUS}}},
false);
# Remove the main manifest so the backup appears aborted
forceStorageRemove(storageRepo(), "${strResumePath}/" . FILE_MANIFEST);
# Add tablespace 2
$oHostDbMaster->manifestTablespaceCreate(\%oManifest, 2);
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768');
@@ -812,10 +815,11 @@ sub run
$oHostDbMaster->manifestTablespaceDrop(\%oManifest, 11);
}
$strTmpPath = $oHostBackup->repoPath() . '/temp/' . $self->stanza() . '.tmp';
# Create resumable backup from last backup
$strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . "/${strBackup}");
testPathMove($oHostBackup->repoPath() . '/backup/' . $self->stanza() . "/${strBackup}", $strTmpPath);
executeTest("sudo chown -R " . $oHostBackup->userGet() . ' ' . dirname($strTmpPath));
# Remove the main manifest so the backup appears aborted
forceStorageRemove(storageRepo(), "${strResumePath}/" . FILE_MANIFEST);
$strBackup = $oHostBackup->backup(
$strType, 'cannot resume - new diff',
@@ -826,14 +830,15 @@ sub run
#-----------------------------------------------------------------------------------------------------------------------
$strType = BACKUP_TYPE_DIFF;
$strTmpPath = $oHostBackup->repoPath() . '/temp/' . $self->stanza() . '.tmp';
# Create resumable backup from last backup
$strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . "/${strBackup}");
testPathMove($oHostBackup->repoPath() . '/backup/' . $self->stanza() . "/${strBackup}", $strTmpPath);
executeTest("sudo chown -R " . $oHostBackup->userGet() . ' ' . dirname($strTmpPath));
# Remove the main manifest so the backup appears aborted
forceStorageRemove(storageRepo(), "${strResumePath}/" . FILE_MANIFEST);
$strBackup = $oHostBackup->backup(
$strType, 'cannot resume - disabled / no repo link',
{oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_NORESUME, bRepoLink => false,
{oExpectedManifest => \%oManifest, strTest => TEST_BACKUP_NORESUME,
strOptionalParam => '--no-resume --log-level-console=detail'});
# Restore
@@ -849,7 +854,7 @@ sub run
# Remap the base and tablespace paths
my %oRemapHash;
$oRemapHash{&MANIFEST_TARGET_PGDATA} = $oHostDbMaster->dbBasePath(2);
filePathCreate($oHostDbMaster->dbBasePath(2));
storageTest()->pathCreate($oHostDbMaster->dbBasePath(2), {strMode => '0700', bCreateParent => true});
$oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/1'} = $oHostDbMaster->tablespacePath(1, 2);
$oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'} = $oHostDbMaster->tablespacePath(2, 2);
@@ -900,7 +905,8 @@ sub run
# Delete the backup.info and make sure the backup fails - the user must then run a stanza-create --force
if ($bNeutralTest)
{
executeTest('sudo rm ' . $oHostBackup->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO);
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT);
}
$oHostDbMaster->manifestFileCreate(
@@ -915,8 +921,8 @@ sub run
strOptionalParam => '--log-level-console=detail'});
# Fail on attempt to create the stanza data since force was not used
$oHostBackup->stanzaCreate('fail on backup directory not empty and missing backup.info',
{iExpectedExitStatus => ERROR_PATH_NOT_EMPTY, strOptionalParam => '--no-' . OPTION_ONLINE});
$oHostBackup->stanzaCreate('fail on backup directory missing backup.info',
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . OPTION_ONLINE});
# Use force to create the stanza
$oHostBackup->stanzaCreate('create required data for stanza',
@@ -1096,7 +1102,7 @@ sub run
executeTest('rm -rf ' . $oHostDbMaster->dbBasePath(2) . "/*");
my $strDbPath = $oHostDbMaster->dbBasePath(2) . '/base';
filePathCreate($strDbPath);
storageTest()->pathCreate($strDbPath, {strMode => '0700'});
$oRemapHash{&MANIFEST_TARGET_PGDATA} = $strDbPath;
delete($oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'});
@@ -1106,7 +1112,7 @@ sub run
'no tablespace remap - error when tablespace dir does not exist', ERROR_PATH_MISSING,
'--log-level-console=detail --tablespace-map-all=../../tablespace', false);
filePathCreate($oHostDbMaster->dbBasePath(2) . '/tablespace');
storageTest()->pathCreate($oHostDbMaster->dbBasePath(2) . '/tablespace', {strMode => '0700'});
$oHostDbMaster->restore(
OPTION_DEFAULT_RESTORE_SET, \%oManifest, undef, $bDelta, $bForce, undef, undef, undef, undef, undef, undef,
@@ -1117,8 +1123,8 @@ sub run
# Backup Info (with an empty stanza)
#-----------------------------------------------------------------------------------------------------------------------
executeTest('sudo chmod g+w ' . $oHostBackup->repoPath() . '/backup');
filePathCreate($oHostBackup->repoPath() . '/backup/db_empty', '0770');
forceStorageMode(storageRepo(), 'backup', 'g+w');
storageRepo()->pathCreate(storageRepo()->pathGet('backup/db_empty'), {strMode => '0770'});
$oHostBackup->info('normal output');
$oHostDbMaster->info('normal output', {strOutput => INFO_OUTPUT_JSON});
@@ -1131,7 +1137,7 @@ sub run
#-----------------------------------------------------------------------------------------------------------------------
if ($bNeutralTest && !$bRemote)
{
executeTest('ls -1R ' . $oHostBackup->repoPath() . '/backup/' . $self->stanza() . '/' . PATH_BACKUP_HISTORY,
executeTest('ls -1R ' . storageRepo()->pathGet('backup/' . $self->stanza() . '/' . PATH_BACKUP_HISTORY),
{oLogTest => $self->expect(), bRemote => $bRemote});
}
test/lib/pgBackRestTest/Module/Info/InfoUnitTest.pm
+17 -33
View File
@@ -21,18 +21,16 @@ use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Info;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Env::ExpireEnvTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::HostEnvTest;
####################################################################################################################################
# initModule
@@ -45,21 +43,6 @@ sub initModule
$self->{strArchivePath} = "$self->{strRepoPath}/archive/" . $self->stanza();
$self->{strBackupPath} = "$self->{strRepoPath}/backup/" . $self->stanza();
$self->{strDbPath} = $self->testPath() . '/db';
# Create the local file object
$self->{oFile} =
new pgBackRest::File
(
$self->stanza(),
$self->{strRepoPath},
new pgBackRest::Protocol::Common::Common
(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
)
);
}
####################################################################################################################################
@@ -70,20 +53,20 @@ sub initTest
my $self = shift;
# Create parent path for pg_control
filePathCreate(($self->{strDbPath} . '/' . DB_PATH_GLOBAL), undef, false, true);
storageTest()->pathCreate(($self->{strDbPath} . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
# Create archive info path
filePathCreate($self->{strArchivePath}, undef, true, true);
storageTest()->pathCreate($self->{strArchivePath}, {bIgnoreExists => true, bCreateParent => true});
# Create backup info path
filePathCreate($self->{strBackupPath}, undef, true, true);
# Create the test object
$self->{oExpireTest} = new pgBackRestTest::Env::ExpireEnvTest(undef, $self->backrestExe(), $self->{oFile}, undef, $self);
storageTest()->pathCreate($self->{strBackupPath}, {bIgnoreExists => true, bCreateParent => true});
# Set options for stanzaCreate
$self->optionStanzaCreate();
# Create the test object
$self->{oExpireTest} = new pgBackRestTest::Env::ExpireEnvTest(undef, $self->backrestExe(), storageRepo(), undef, $self);
$self->{oExpireTest}->stanzaCreate($self->stanza(), PG_VERSION_94);
}
@@ -93,6 +76,7 @@ sub optionStanzaCreate
# Set options for stanzaCreate
my $oOption = {};
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_DB_PATH, $self->{strDbPath});
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->{strRepoPath});
@@ -113,7 +97,7 @@ sub run
my $oOption = {};
# $self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->{strRepoPath});
# Used to create backups and WAL to test
@@ -123,18 +107,18 @@ sub run
################################################################################################################################
if ($self->begin("Info->formatTextStanza() && Info->formatTextBackup()"))
{
$self->configLoadExpect(dclone($oOption), CMD_INFO);
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_INFO); logEnable();
my $oInfo = new pgBackRest::Info();
#---------------------------------------------------------------------------------------------------------------------------
my $hyStanza = $oInfo->stanzaList($self->{oFile}, $self->stanza());
my $hyStanza = $oInfo->stanzaList($self->stanza());
$self->testResult(sub {$oInfo->formatTextStanza(@{$hyStanza}[0])},
"stanza: db\n status: error (no valid backups)\n wal archive min/max: none present", "stanza text output");
#---------------------------------------------------------------------------------------------------------------------------
$self->{oExpireTest}->backupCreate($self->stanza(), BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY, -1, -1);
$hyStanza = $oInfo->stanzaList($self->{oFile}, $self->stanza());
$hyStanza = $oInfo->stanzaList($self->stanza());
$self->testResult(sub {$oInfo->formatTextStanza(@{$hyStanza}[-1])},
"stanza: db\n status: ok\n wal archive min/max: none present",
@@ -150,7 +134,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
$self->{oExpireTest}->backupCreate($self->stanza(), BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY);
$hyStanza = $oInfo->stanzaList($self->{oFile}, $self->stanza());
$hyStanza = $oInfo->stanzaList($self->stanza());
$self->testResult(sub {$oInfo->formatTextStanza(@{$hyStanza}[-1])},
"stanza: db\n status: ok\n wal archive min/max: 000000010000000000000000 / 000000010000000000000005",
@@ -166,7 +150,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
$self->{oExpireTest}->backupCreate($self->stanza(), BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY, 256);
$hyStanza = $oInfo->stanzaList($self->{oFile}, $self->stanza());
$hyStanza = $oInfo->stanzaList($self->stanza());
$self->testResult(sub {$oInfo->formatTextStanza(@{$hyStanza}[-1])},
"stanza: db\n status: ok\n wal archive min/max: 000000010000000000000000 / 000000010000000100000008",
@@ -185,7 +169,7 @@ sub run
$self->optionStanzaCreate();
$self->{oExpireTest}->stanzaUpgrade($self->stanza(), PG_VERSION_95);
$self->{oExpireTest}->backupCreate($self->stanza(), BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY, 2);
$hyStanza = $oInfo->stanzaList($self->{oFile}, $self->stanza());
$hyStanza = $oInfo->stanzaList($self->stanza());
$self->testResult(sub {$oInfo->formatTextStanza(@{$hyStanza}[-1])},
"stanza: db\n status: ok\n wal archive min/max: 000000010000000000000000 / 000000010000000000000004",
test/lib/pgBackRestTest/Module/Protocol/ProtocolCommonMinionTest.pm
+132
View File
@@ -0,0 +1,132 @@
####################################################################################################################################
# ProtocolCommonMinionTest.pm - tests for Protocol::Common::Minion module
####################################################################################################################################
package pgBackRestTest::Module::Protocol::ProtocolCommonMinionTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use IO::Socket::UNIX;
use Time::HiRes qw(usleep);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Protocol::Base::Minion;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# socketServer
####################################################################################################################################
sub socketServer
{
my $self = shift;
my $strSocketFile = shift;
my $fnServer = shift;
# Fork off the server
if (fork() == 0)
{
# Open the domain socket
my $oSocketServer = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Local => $strSocketFile, Listen => 1);
&log(INFO, " * socket server open");
# Wait for a connection and create IO object
my $oConnection = $oSocketServer->accept();
my $oIoHandle = new pgBackRest::Common::Io::Handle('socket server', $oConnection, $oConnection);
&log(INFO, " * socket server connected");
# Run server function
$fnServer->($oIoHandle);
# Shutdown server
$oSocketServer->close();
&log(INFO, " * socket server closed");
unlink($strSocketFile);
exit 0;
}
# Wait for client socket
my $oWait = waitInit(5);
while (!-e $strSocketFile && waitMore($oWait)) {};
# Open the client socket
my $oClient = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Peer => $strSocketFile);
if (!defined($oClient))
{
logErrorResult(ERROR_FILE_OPEN, 'unable to open client socket', $OS_ERROR);
}
&log(INFO, " * socket client connected");
return $oClient;
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strSocketFile = $self->testPath() . qw{/} . 'domain.socket';
################################################################################################################################
# if ($self->begin('new() & timeout() & bufferMax()'))
# {
# #---------------------------------------------------------------------------------------------------------------------------
# my $oIoBuffered = $self->testResult(
# sub {new pgBackRest::Common::Io::Buffered(
# new pgBackRest::Common::Io::Handle('test'), 10, 2048)}, '[object]', 'new - no handles');
#
# $self->testResult(sub {$oIoBuffered->timeout()}, 10, ' check timeout');
# $self->testResult(sub {$oIoBuffered->bufferMax()}, 2048, ' check buffer max');
# }
################################################################################################################################
if ($self->begin('process()'))
{
my $oClient = $self->socketServer($strSocketFile, sub
{
my $oIoHandle = shift;
my $oMinion = new pgBackRest::Protocol::Base::Minion('test', new pgBackRest::Common::Io::Buffered($oIoHandle, 5, 4096));
$oMinion->process();
});
#---------------------------------------------------------------------------------------------------------------------------
my $oIoBuffered = $self->testResult(
sub {new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('socket client', $oClient, $oClient), 5, 4096)}, '[object]', 'open');
$self->testResult(
sub {$oIoBuffered->readLine()},
'{"name":"' . BACKREST_NAME . '","service":"test","version":"' . BACKREST_VERSION . '"}', 'read greeting');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oIoBuffered->writeLine('{"cmd":"noop"}')}, 15, 'write noop');
$self->testResult(
sub {$oIoBuffered->readLine()}, '{"out":[]}', 'read output');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oIoBuffered->writeLine('{"cmd":"exit"}')}, 15, 'write exit');
$self->testResult(
sub {$oIoBuffered->readLine(true)}, undef, 'read EOF');
}
}
1;
test/lib/pgBackRestTest/Module/Stanza/StanzaCreateTest.pm
+61 -42
View File
@@ -21,13 +21,16 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Filter::Gzip;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
@@ -37,13 +40,13 @@ sub run
{
my $self = shift;
for (my $bRemote = false; $bRemote <= true; $bRemote++)
foreach my $bRemote (false, true)
{
# Increment the run, log, and decide whether this unit test should be run
if (!$self->begin($bRemote ? "remote" : "local")) {next}
if (!$self->begin("remote ${bRemote}")) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote});
# Create the stanza
@@ -51,12 +54,12 @@ sub run
strOptionalParam => '--no-' . OPTION_ONLINE});
# Create the test path for pg_control
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
storageDb()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
# Copy pg_control for stanza-create
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
storageDb()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin',
$oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
$oHostBackup->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-' . OPTION_ONLINE});
@@ -65,31 +68,38 @@ sub run
# Create the xlog path
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
filePathCreate($strXlogPath, undef, false, true);
storageDb()->pathCreate($strXlogPath, {bCreateParent => true});
# Generate WAL then push to get valid archive data in the archive directory
my ($strArchiveFile, $strSourceFile) = $self->archiveGenerate($oFile, $strXlogPath, 1, 1, WAL_VERSION_94);
my ($strArchiveFile, $strSourceFile) = $self->archiveGenerate($strXlogPath, 1, 1, WAL_VERSION_94);
my $strCommand = $oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() .
' --stanza=db archive-push';
$oHostDbMaster->executeSimple($strCommand . " ${strSourceFile}", {oLogTest => $self->expect()});
# With data existing in the archive dir, remove the info file and confirm failure
$oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
# With data existing in the archive dir, remove the info files and confirm failure
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE);
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT);
$oHostBackup->stanzaCreate('fail on archive info file missing from non-empty dir',
{iExpectedExitStatus => ERROR_PATH_NOT_EMPTY, strOptionalParam => '--no-' . OPTION_ONLINE});
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . OPTION_ONLINE});
# Change the permissions of the archive file so it cannot be read
executeTest('sudo chmod 220 ' . $oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
substr($strArchiveFile, 0, 16) . "/*.gz");
forceStorageMode(
storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' .
COMPRESS_EXT,
'220');
# Force creation of the info file but fail on gunzip
$oHostBackup->stanzaCreate('gunzip fail on forced stanza-create',
{iExpectedExitStatus => ERROR_FILE_OPEN, strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
# Change permissions back and force creation of archive info from the gz file
executeTest('sudo chmod 640 ' . $oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
substr($strArchiveFile, 0, 16) . "/*.gz");
# Change permissions back
forceStorageMode(
storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' .
COMPRESS_EXT,
'640');
# Force creation of archive info from the gz file
$oHostBackup->stanzaCreate('force create archive.info from gz file',
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
@@ -97,16 +107,9 @@ sub run
# stanza already exists
$oHostBackup->stanzaCreate('repeat create', {strOptionalParam => '--no-' . OPTION_ONLINE});
# Remove the backup info file and confirm success with backup dir empty
# Backup Full tests will confirm failure when backup dir not empty
$oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
$oHostBackup->stanzaCreate('force not needed when backup dir empty, archive.info exists but backup.info is missing',
{strOptionalParam => '--no-' . OPTION_ONLINE});
# Remove the backup.info file then munge and save the archive info file
$oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
# Munge and save the archive info file
$oHostBackup->infoMunge(
$oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE),
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_DB_VERSION => '8.0'}});
$oHostBackup->stanzaCreate('hash check fails requiring force',
@@ -116,38 +119,54 @@ sub run
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
# Cleanup the global hash but don't save the file (permission issues may prevent it anyway)
$oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE), false);
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), false);
# Change the database version by copying a new pg_control file
executeTest('sudo rm ' . $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
executeTest('cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_93 . '.bin ' . $oHostDbMaster->dbBasePath() .
'/' . DB_FILE_PGCONTROL);
storageDb()->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
storageDb()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_93 . '.bin',
$oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
$oHostBackup->stanzaCreate('fail on database mismatch without force option',
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . OPTION_ONLINE});
# Restore pg_control
executeTest('sudo rm ' . $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
executeTest('cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' .
storageDb()->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
storageDb()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin',
$oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
# Unzip the archive file and recreate the archive.info file from it
executeTest('sudo gunzip ' . $oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
substr($strArchiveFile, 0, 16) . "/${strArchiveFile}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27.gz");
my $strArchiveTest = PG_VERSION_94 . "-1/${strArchiveFile}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27";
forceStorageMode(
storageRepo(), dirname(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveTest}.gz")), 'g+w',
{bRecursive => true});
storageRepo()->copy(
storageRepo()->openRead(
STORAGE_REPO_ARCHIVE . "/${strArchiveTest}.gz",
{rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}]}),
STORAGE_REPO_ARCHIVE . "/${strArchiveTest}");
$oHostBackup->stanzaCreate('force create archive.info from uncompressed file',
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
# Remove the uncompressed WAL archive file and archive.info
executeTest('sudo rm ' . $oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
substr($strArchiveFile, 0, 16) . "/${strArchiveFile}-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27");
$oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . "/${strArchiveTest}");
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE);
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT);
$oHostBackup->stanzaCreate('force with missing WAL archive file',
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
# Remove the WAL archive directory
executeTest('sudo rm -rf ' . $oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
substr($strArchiveFile, 0, 16));
$oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
forceStorageRemove(
storageRepo(),
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16), {bRecurse => true});
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE);
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT);
$oHostBackup->stanzaCreate('force with missing WAL archive directory',
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
}
test/lib/pgBackRestTest/Module/Stanza/StanzaUnitTest.pm
+264 -23
View File
@@ -19,19 +19,19 @@ use pgBackRest::Archive::ArchiveCommon;
use pgBackRest::Archive::ArchiveInfo;
use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Common::Common;
use pgBackRest::Protocol::Helper;
use pgBackRest::Stanza;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::RunTest;
@@ -57,13 +57,13 @@ sub initTest
my $self = shift;
# Create archive info path
filePathCreate($self->{strArchivePath}, undef, true, true);
storageTest()->pathCreate($self->{strArchivePath}, {bIgnoreExists => true, bCreateParent => true});
# Create backup info path
filePathCreate($self->{strBackupPath}, undef, true, true);
storageTest()->pathCreate($self->{strBackupPath}, {bIgnoreExists => true, bCreateParent => true});
# Create pg_control path
filePathCreate(($self->{strDbPath} . '/' . DB_PATH_GLOBAL), undef, false, true);
storageTest()->pathCreate($self->{strDbPath} . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
# Copy a pg_control file into the pg_control path
executeTest(
@@ -90,15 +90,255 @@ sub run
$self->optionSetTest($oOption, OPTION_DB_TIMEOUT, 5);
$self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 6);
# ??? Currently only contains unit tests for stanza-upgrade. TODO stanza-create
################################################################################################################################
if ($self->begin("Stanza::new"))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->optionBoolSetTest($oOption, OPTION_ONLINE, true);
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
# my $oStanza = new pgBackRest::Stanza();
$self->testException(sub {(new pgBackRest::Stanza())}, ERROR_DB_CONNECT,
"could not connect to server: No such file or directory\n");
$self->optionBoolSetTest($oOption, OPTION_ONLINE, false);
}
################################################################################################################################
if ($self->begin("Stanza::stanzaUpgrade"))
if ($self->begin("Stanza::process()"))
{
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false);
#---------------------------------------------------------------------------------------------------------------------------
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_CHECK); logEnable();
my $oStanza = new pgBackRest::Stanza();
$self->testException(sub {$oStanza->process()}, ERROR_ASSERT,
"stanza->process() called with invalid command: " . CMD_CHECK);
#---------------------------------------------------------------------------------------------------------------------------
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
rmdir($self->{strArchivePath});
rmdir($self->{strBackupPath});
$self->testResult(sub {$oStanza->process()}, 0, 'parent paths recreated successfully');
}
################################################################################################################################
if ($self->begin("Stanza::stanzaCreate()"))
{
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
my $oStanza = new pgBackRest::Stanza();
my $strBackupInfoFile = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO);
my $strBackupInfoFileCopy = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT);
my $strArchiveInfoFile = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE);
# No force. Archive dir not empty. No archive.info file. Backup directory empty.
#---------------------------------------------------------------------------------------------------------------------------
storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE . "/9.4-1");
$self->testException(sub {$oStanza->stanzaCreate()}, ERROR_PATH_NOT_EMPTY,
"archive directory not empty" .
"\nHINT: use stanza-create --force to force the stanza data to be created.");
# No force. Archive dir not empty. No archive.info file. Backup directory not empty. No backup.info file.
#---------------------------------------------------------------------------------------------------------------------------
storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/12345");
$self->testException(sub {$oStanza->stanzaCreate()}, ERROR_PATH_NOT_EMPTY,
"backup directory and/or archive directory not empty" .
"\nHINT: use stanza-create --force to force the stanza data to be created.");
# No force. Archive dir empty. No archive.info file. Backup directory not empty. No backup.info file.
#---------------------------------------------------------------------------------------------------------------------------
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . "/9.4-1", {bRecurse => true});
$self->testException(sub {$oStanza->stanzaCreate()}, ERROR_PATH_NOT_EMPTY,
"backup directory not empty" .
"\nHINT: use stanza-create --force to force the stanza data to be created.");
# No force. No archive.info file and no archive sub-directories or files. Backup.info exists and no backup sub-directories
# or files
#---------------------------------------------------------------------------------------------------------------------------
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . "/12345", {bRecurse => true});
(new pgBackRest::Backup::Info($self->{strBackupPath}, false, false, {bIgnoreMissing => true}))->create(PG_VERSION_94,
WAL_VERSION_94_SYS_ID, '942', '201409291', true);
$self->testException(sub {$oStanza->stanzaCreate()}, ERROR_FILE_MISSING,
"archive information missing" .
"\nHINT: use stanza-create --force to force the stanza data to be created.");
# No force. No backup.info file (backup.info.copy only) and no backup sub-directories or files. Archive.info exists and no
# archive sub-directories or files
#---------------------------------------------------------------------------------------------------------------------------
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO);
(new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false, {bIgnoreMissing => true}))->create(PG_VERSION_94,
WAL_VERSION_94_SYS_ID, true);
$self->testException(sub {$oStanza->stanzaCreate()}, ERROR_FILE_MISSING,
"backup information missing" .
"\nHINT: use stanza-create --force to force the stanza data to be created.");
# No force. Valid archive.info exists. Invalid backup.info exists.
#---------------------------------------------------------------------------------------------------------------------------
(new pgBackRest::Backup::Info($self->{strBackupPath}, false, false))->create(PG_VERSION_94, WAL_VERSION_93_SYS_ID, '942',
'201409291', true);
$self->testException(sub {$oStanza->stanzaCreate()}, ERROR_FILE_INVALID,
"backup info file invalid" .
"\nHINT: use stanza-upgrade if the database has been upgraded or use --force");
# No force. Invalid archive.info exists. Invalid backup.info exists.
#---------------------------------------------------------------------------------------------------------------------------
(new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false))->create(PG_VERSION_94, WAL_VERSION_93_SYS_ID, true);
$self->testException(sub {$oStanza->stanzaCreate()}, ERROR_FILE_INVALID,
"archive info file invalid" .
"\nHINT: use stanza-upgrade if the database has been upgraded or use --force");
# Create stanza without force
#---------------------------------------------------------------------------------------------------------------------------
forceStorageRemove(storageRepo(), $strBackupInfoFile . "*");
forceStorageRemove(storageRepo(), $strArchiveInfoFile . "*");
$self->testResult(sub {$oStanza->stanzaCreate()}, 0, 'successfully created stanza without force');
# Create stanza successfully with .info and .info.copy files already existing
#--------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oStanza->stanzaCreate()}, 0, 'successfully created stanza without force with existing info files');
# Remove only backup.info.copy file - confirm stanza create does not throw an error
#---------------------------------------------------------------------------------------------------------------------------
forceStorageRemove(storageRepo(), $strBackupInfoFileCopy);
$self->testResult(sub {$oStanza->stanzaCreate()}, 0, 'no error on missing copy file');
$self->testResult(sub {storageRepo()->exists($strBackupInfoFile) &&
!storageRepo()->exists($strBackupInfoFileCopy)},
true, ' and only backup.info exists');
# Force on, Repo-Sync off. Archive dir empty. No archive.info file. Backup directory not empty. No backup.info file.
#---------------------------------------------------------------------------------------------------------------------------
$self->optionBoolSetTest($oOption, OPTION_FORCE, true);
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
forceStorageRemove(storageRepo(), $strBackupInfoFile . "*");
forceStorageRemove(storageRepo(), $strArchiveInfoFile . "*");
storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/12345");
$oStanza = new pgBackRest::Stanza();
$self->testResult(sub {$oStanza->stanzaCreate()}, 0, 'successfully created stanza with force');
$self->testResult(sub {(new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}))->check(PG_VERSION_94,
WAL_VERSION_94_SYS_ID) && (new pgBackRest::Backup::Info($self->{strBackupPath}))->check(PG_VERSION_94, '942',
'201409291', WAL_VERSION_94_SYS_ID)}, 1, ' new info files correct');
$self->optionReset($oOption, OPTION_FORCE);
}
################################################################################################################################
if ($self->begin("Stanza::infoFileCreate"))
{
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
my $oStanza = new pgBackRest::Stanza();
my @stryFileList = ('anything');
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false, {bIgnoreMissing => true});
my $oBackupInfo = new pgBackRest::Backup::Info($self->{strBackupPath}, false, false, {bIgnoreMissing => true});
# If infoFileCreate is ever called directly, confirm it errors if something other than the info file exists in archive dir
# when --force is not used
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oStanza->infoFileCreate($oArchiveInfo, STORAGE_REPO_ARCHIVE, $self->{strArchivePath},
\@stryFileList)}, ERROR_PATH_NOT_EMPTY,
"archive directory not empty" .
"\nHINT: use stanza-create --force to force the stanza data to be created.");
# If infoFileCreate is ever called directly, confirm it errors if something other than the info file exists in backup dir
# when --force is not used
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oStanza->infoFileCreate($oBackupInfo, STORAGE_REPO_BACKUP, $self->{strBackupPath},
\@stryFileList)}, ERROR_PATH_NOT_EMPTY,
"backup directory not empty" .
"\nHINT: use stanza-create --force to force the stanza data to be created.");
# Set force option --------
$self->optionBoolSetTest($oOption, OPTION_FORCE, true);
# Force. Invalid archive.info exists.
#---------------------------------------------------------------------------------------------------------------------------
$self->optionBoolSetTest($oOption, OPTION_FORCE, true);
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
$oArchiveInfo->create(PG_VERSION_94, 12345, true);
$oStanza = new pgBackRest::Stanza();
$self->testResult(sub {$oStanza->infoFileCreate($oArchiveInfo, STORAGE_REPO_ARCHIVE, $self->{strArchivePath},
\@stryFileList)}, "(0, [undef])", 'force successful for invalid info file');
# Cause an error to be thrown by changing the permissions of the archive file so it cannot be read for the hash comparison
#---------------------------------------------------------------------------------------------------------------------------
executeTest('sudo chmod 220 ' . $self->{strArchivePath} . "/archive.info");
$self->testResult(sub {$oStanza->infoFileCreate($oArchiveInfo, STORAGE_REPO_ARCHIVE, $self->{strArchivePath},
\@stryFileList)},
"(" . &ERROR_FILE_OPEN . ", unable to open '" . $self->{strArchivePath} . "/archive.info': Permission denied)",
'exception code path');
executeTest('sudo chmod 640 ' . $self->{strArchivePath} . "/archive.info");
# Force. Archive dir not empty. Warning returned.
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->pathCreate($self->{strArchivePath} . "/9.3-0", {bIgnoreExists => true, bCreateParent => true});
$self->testResult(sub {$oStanza->infoFileCreate($oArchiveInfo, STORAGE_REPO_ARCHIVE, $self->{strArchivePath},
\@stryFileList)}, "(0, [undef])", 'force successful with archive.info file warning',
{strLogExpect => "WARN: found empty directory " . $self->{strArchivePath} . "/9.3-0"});
# Reset force option --------
$self->optionReset($oOption, OPTION_FORCE);
}
################################################################################################################################
if ($self->begin("Stanza::infoObject()"))
{
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_UPGRADE); logEnable();
my $oStanza = new pgBackRest::Stanza();
$self->testException(sub {$oStanza->infoObject(STORAGE_REPO_BACKUP, $self->{strBackupPath})}, ERROR_FILE_MISSING,
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO) .
" does not exist and is required to perform a backup." .
"\nHINT: has a stanza-create been performed?");
# Force valid but not set.
#---------------------------------------------------------------------------------------------------------------------------
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
$oStanza = new pgBackRest::Stanza();
$self->testException(sub {$oStanza->infoObject(STORAGE_REPO_BACKUP, $self->{strBackupPath})}, ERROR_FILE_MISSING,
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO) .
" does not exist and is required to perform a backup." .
"\nHINT: has a stanza-create been performed?" .
"\nHINT: use stanza-create --force to force the stanza data to be created.");
# Force.
#---------------------------------------------------------------------------------------------------------------------------
$self->optionBoolSetTest($oOption, OPTION_FORCE, true);
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
$self->testResult(sub {$oStanza->infoObject(STORAGE_REPO_ARCHIVE, $self->{strArchivePath})}, "[object]",
'archive force successful');
$self->testResult(sub {$oStanza->infoObject(STORAGE_REPO_BACKUP, $self->{strBackupPath})}, "[object]",
'backup force successful');
# Reset force option --------
$self->optionReset($oOption, OPTION_FORCE);
# Cause an error to be thrown by changing the permissions of the archive file so it cannot be read
#---------------------------------------------------------------------------------------------------------------------------
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); logEnable();
(new pgBackRest::Backup::Info($self->{strBackupPath}, false, false, {bIgnoreMissing => true}))->create(PG_VERSION_94,
WAL_VERSION_94_SYS_ID, '942', '201409291', true);
forceStorageRemove(storageRepo(), storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT));
executeTest('sudo chmod 220 ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO));
$self->testException(sub {$oStanza->infoObject(STORAGE_REPO_BACKUP, $self->{strBackupPath})}, ERROR_FILE_OPEN,
"unable to open '" . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO) .
"': Permission denied");
executeTest('sudo chmod 640 ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO));
}
################################################################################################################################
if ($self->begin("Stanza::stanzaUpgrade()"))
{
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_UPGRADE); logEnable();
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false, {bIgnoreMissing => true});
$oArchiveInfo->create('9.3', '6999999999999999999', true);
my $oBackupInfo = new pgBackRest::Backup::Info($self->{strBackupPath}, false, false);
my $oBackupInfo = new pgBackRest::Backup::Info($self->{strBackupPath}, false, false, {bIgnoreMissing => true});
$oBackupInfo->create('9.3', '6999999999999999999', '937', '201306121', true);
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_UPGRADE); logEnable();
@@ -112,41 +352,41 @@ sub run
}
################################################################################################################################
if ($self->begin("Stanza::upgradeCheck"))
if ($self->begin("Stanza::upgradeCheck()"))
{
logDisable(); $self->configLoadExpect(dclone($oOption), CMD_STANZA_UPGRADE); logEnable();
my $oStanza = new pgBackRest::Stanza();
# Create the archive file with current data
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false);
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath}, false, {bIgnoreMissing => true});
$oArchiveInfo->create('9.4', 6353949018581704918, true);
# Create the backup file with outdated data
my $oBackupInfo = new pgBackRest::Backup::Info($self->{strBackupPath}, false, false);
my $oBackupInfo = new pgBackRest::Backup::Info($self->{strBackupPath}, false, false, {bIgnoreMissing => true});
$oBackupInfo->create('9.3', 6999999999999999999, '937', '201306121', true);
# Confirm upgrade is needed for backup
$self->testResult(sub {$oStanza->upgradeCheck($oBackupInfo, PATH_BACKUP_CLUSTER, ERROR_BACKUP_MISMATCH)}, true,
$self->testResult(sub {$oStanza->upgradeCheck($oBackupInfo, STORAGE_REPO_BACKUP, ERROR_BACKUP_MISMATCH)}, true,
'backup upgrade needed');
$self->testResult(sub {$oStanza->upgradeCheck($oArchiveInfo, PATH_BACKUP_ARCHIVE, ERROR_ARCHIVE_MISMATCH)}, false,
$self->testResult(sub {$oStanza->upgradeCheck($oArchiveInfo, STORAGE_REPO_ARCHIVE, ERROR_ARCHIVE_MISMATCH)}, false,
'archive upgrade not needed');
# Change archive file to contain outdated data
$oArchiveInfo->create('9.3', 6999999999999999999, true);
# Confirm upgrade is needed for both
$self->testResult(sub {$oStanza->upgradeCheck($oArchiveInfo, PATH_BACKUP_ARCHIVE, ERROR_ARCHIVE_MISMATCH)}, true,
$self->testResult(sub {$oStanza->upgradeCheck($oArchiveInfo, STORAGE_REPO_ARCHIVE, ERROR_ARCHIVE_MISMATCH)}, true,
'archive upgrade needed');
$self->testResult(sub {$oStanza->upgradeCheck($oBackupInfo, PATH_BACKUP_CLUSTER, ERROR_BACKUP_MISMATCH)}, true,
$self->testResult(sub {$oStanza->upgradeCheck($oBackupInfo, STORAGE_REPO_BACKUP, ERROR_BACKUP_MISMATCH)}, true,
'backup upgrade needed');
# Change the backup file to contain current data
$oBackupInfo->create('9.4', 6353949018581704918, '942', '201409291', true);
# Confirm upgrade is needed for archive
$self->testResult(sub {$oStanza->upgradeCheck($oBackupInfo, PATH_BACKUP_CLUSTER, ERROR_BACKUP_MISMATCH)}, false,
$self->testResult(sub {$oStanza->upgradeCheck($oBackupInfo, STORAGE_REPO_BACKUP, ERROR_BACKUP_MISMATCH)}, false,
'backup upgrade not needed');
$self->testResult(sub {$oStanza->upgradeCheck($oArchiveInfo, PATH_BACKUP_ARCHIVE, ERROR_ARCHIVE_MISMATCH)}, true,
$self->testResult(sub {$oStanza->upgradeCheck($oArchiveInfo, STORAGE_REPO_ARCHIVE, ERROR_ARCHIVE_MISMATCH)}, true,
'archive upgrade needed');
#---------------------------------------------------------------------------------------------------------------------------
@@ -156,9 +396,9 @@ sub run
$oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{strArchivePath});
$oBackupInfo = new pgBackRest::Backup::Info($self->{strBackupPath});
$self->testResult(sub {$oStanza->upgradeCheck($oArchiveInfo, PATH_BACKUP_ARCHIVE, ERROR_ARCHIVE_MISMATCH)}, false,
$self->testResult(sub {$oStanza->upgradeCheck($oArchiveInfo, STORAGE_REPO_ARCHIVE, ERROR_ARCHIVE_MISMATCH)}, false,
'archive upgrade not necessary');
$self->testResult(sub {$oStanza->upgradeCheck($oBackupInfo, PATH_BACKUP_CLUSTER, ERROR_BACKUP_MISMATCH)}, false,
$self->testResult(sub {$oStanza->upgradeCheck($oBackupInfo, STORAGE_REPO_BACKUP, ERROR_BACKUP_MISMATCH)}, false,
'backup upgrade not necessary');
#---------------------------------------------------------------------------------------------------------------------------
@@ -167,11 +407,12 @@ sub run
$oStanza->{oDb}{ullDbSysId} = 6999999999999999999;
# Pass an expected error that is different than the actual error and confirm an error is thrown
$self->testException(sub {$oStanza->upgradeCheck($oArchiveInfo, PATH_BACKUP_ARCHIVE, ERROR_ASSERT)}, ERROR_ARCHIVE_MISMATCH,
$self->testException(sub {$oStanza->upgradeCheck($oArchiveInfo, STORAGE_REPO_ARCHIVE, ERROR_ASSERT)},
ERROR_ARCHIVE_MISMATCH,
"WAL segment version 9.3 does not match archive version 9.4\n" .
"WAL segment system-id 6999999999999999999 does not match archive system-id 6353949018581704918\n" .
"HINT: are you archiving to the correct stanza?");
$self->testException(sub {$oStanza->upgradeCheck($oBackupInfo, PATH_BACKUP_CLUSTER, ERROR_ASSERT)}, ERROR_BACKUP_MISMATCH,
$self->testException(sub {$oStanza->upgradeCheck($oBackupInfo, STORAGE_REPO_BACKUP, ERROR_ASSERT)}, ERROR_BACKUP_MISMATCH,
"database version = 9.3, system-id 6999999999999999999 does not match backup version = 9.4, " .
"system-id = 6353949018581704918\nHINT: is this the correct stanza?");
}
test/lib/pgBackRestTest/Module/Stanza/StanzaUpgradeTest.pm
+36 -30
View File
@@ -20,19 +20,20 @@ use File::Basename qw(dirname);
use pgBackRest::Archive::ArchiveInfo;
use pgBackRest::Backup::Info;
use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::DbVersion;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
@@ -48,20 +49,20 @@ sub run
if (!$self->begin($bRemote ? "remote" : "local")) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile) = $self->setup(
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote});
# Create the test path for pg_control
filePathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), undef, false, true);
storageDb()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
# Copy pg_control for stanza-create
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_93 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
storageDb()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_93 . '.bin',
$oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
# Create the xlog path for pushing WAL
my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
filePathCreate($strXlogPath, undef, false, true);
storageDb()->pathCreate($strXlogPath, {bCreateParent => true});
my $strArchiveTestFile = $self->dataPath() . '/backup.wal1_';
# Attempt an upgrade before stanza-create has been performed
@@ -79,7 +80,9 @@ sub run
# Fail upgrade when backup.info missing
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO);
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT);
$oHostBackup->stanzaUpgrade('fail on stanza not initialized since backup.info is missing',
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . OPTION_ONLINE});
@@ -90,14 +93,11 @@ sub run
# Fail on archive push due to mismatch of DB since stanza not upgraded
#--------------------------------------------------------------------------------------------------------------------------
# Push a WAL segment so have a valid file in archive dir
# $oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile . WAL_VERSION_93 . '.bin', 1);
# Upgrade the DB by copying new pg_control
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
executeTest('sudo chmod 600 ' . $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
storageDb()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_94 . '.bin',
$oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
# Fail on attempt to push an archive
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile . WAL_VERSION_94 . '.bin', 1, ERROR_ARCHIVE_MISMATCH);
@@ -111,43 +111,49 @@ sub run
# Push a WAL segment so have a valid file in the latest DB archive dir only
$oHostDbMaster->archivePush($strXlogPath, $strArchiveTestFile . WAL_VERSION_94 . '.bin', 1);
$self->testResult(
sub {$oFile->list(PATH_BACKUP_ARCHIVE, PG_VERSION_94 . '-2/0000000100000001')},
"000000010000000100000001-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27.$oFile->{strCompressExtension}",
sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-2/0000000100000001')},
"000000010000000100000001-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27." . COMPRESS_EXT,
'check that WAL is in the archive at -2');
# Create a DB history mismatch between the info files
#--------------------------------------------------------------------------------------------------------------------------
# Remove the archive info file and force reconstruction
$oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE));
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE);
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT);
$oHostBackup->stanzaCreate('use force to recreate the stanza producing mismatched info history but same current db-id',
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
# Create a DB-ID mismatch between the info files
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO));
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO);
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT);
$oHostBackup->stanzaCreate('use force to recreate the stanza producing mismatched db-id',
{strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
# Confirm successful backup at db-1 although archive at db-2
#--------------------------------------------------------------------------------------------------------------------------
# Create the tablespace directory and perform a backup
filePathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_PGTBLSPC);
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_PGTBLSPC);
$oHostBackup->backup('full', 'create first full backup ', {strOptionalParam => '--retention-full=2 --no-' .
OPTION_ONLINE . ' --log-level-console=detail'}, false);
# Test archive dir version XX.Y-Z ensuring sort order of db ids is reconstructed correctly from the directory db-id value
#--------------------------------------------------------------------------------------------------------------------------
# Create the 10.0-3 directory and copy a WAL file to it (something that has a different system id)
$oHostBackup->executeSimple('mkdir ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE) . '/' . '10.0-3');
$oHostBackup->executeSimple('mkdir ' . $oFile->pathGet(PATH_BACKUP_ARCHIVE) . '/' . '10.0-3/0000000100000001');
$oHostBackup->executeSimple('cp ' . $self->dataPath() . '/backup.wal1_' . WAL_VERSION_92 . '.bin '
. $oFile->pathGet(PATH_BACKUP_ARCHIVE) . '/' . '10.0-3/0000000100000001/000000010000000100000001');
forceStorageMode(storageRepo(), STORAGE_REPO_ARCHIVE, '770');
storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE . '/10.0-3/0000000100000001', {bCreateParent => true});
storageRepo()->copy(
storageDb()->openRead($self->dataPath() . '/backup.wal1_' . WAL_VERSION_92 . '.bin'),
STORAGE_REPO_ARCHIVE . '/10.0-3/0000000100000001/000000010000000100000001');
forceStorageOwner(storageRepo(), STORAGE_REPO_ARCHIVE . '/10.0-3', $oHostBackup->userGet(), {bRecurse => true});
# Copy pg_control for 9.5
executeTest(
'cp ' . $self->dataPath() . '/backup.pg_control_' . WAL_VERSION_95 . '.bin ' . $oHostDbMaster->dbBasePath() . '/' .
DB_FILE_PGCONTROL);
executeTest('sudo chmod 600 ' . $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
storageDb()->copy(
$self->dataPath() . '/backup.pg_control_' . WAL_VERSION_95 . '.bin',
$oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
$oHostBackup->stanzaUpgrade('successfully upgrade with XX.Y-Z', {strOptionalParam => '--no-' . OPTION_ONLINE});
test/lib/pgBackRestTest/Module/Storage/StorageFilterGzipTest.pm
+197
View File
@@ -0,0 +1,197 @@
####################################################################################################################################
# StorageFilterGzipTest.pm - Tests for Storage::Filter::Gzip module.
####################################################################################################################################
package pgBackRestTest::Module::Storage::StorageFilterGzipTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Compress::Raw::Zlib qw(Z_OK Z_BUF_ERROR Z_DATA_ERROR);
use Digest::SHA qw(sha1_hex);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Filter::Gzip;
use pgBackRest::Storage::Posix::Driver;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strFile = $self->testPath() . qw{/} . 'file.txt';
my $strFileGz = "${strFile}.gz";
my $strFileContent = 'TESTDATA';
my $iFileLength = length($strFileContent);
my $oDriver = new pgBackRest::Storage::Posix::Driver();
################################################################################################################################
if ($self->begin('errorCheck()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz))}, '[object]', 'new write');
$oGzipIo->{bWrite} = true;
$self->testException(sub {$oGzipIo->errorCheck(Z_DATA_ERROR)}, ERROR_FILE_WRITE, "unable to deflate '${strFileGz}'");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oGzipIo->errorCheck(Z_OK)}, Z_OK, 'Z_OK');
$self->testResult(sub {$oGzipIo->errorCheck(Z_BUF_ERROR)}, Z_OK, 'Z_BUF_ERROR');
#---------------------------------------------------------------------------------------------------------------------------
$oGzipIo->{bWrite} = false;
$oGzipIo->{strCompressType} = STORAGE_DECOMPRESS;
$self->testException(sub {$oGzipIo->errorCheck(Z_DATA_ERROR)}, ERROR_FILE_READ, "unable to inflate '${strFileGz}'");
}
################################################################################################################################
if ($self->begin('write()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz))}, '[object]', 'new write compress');
my $tBuffer = substr($strFileContent, 0, 2);
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes');
$tBuffer = substr($strFileContent, 2, 2);
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes');
$tBuffer = substr($strFileContent, 4, 2);
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes');
$tBuffer = substr($strFileContent, 6, 2);
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 2, ' write 2 bytes');
$tBuffer = '';
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 0, ' write 0 bytes');
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
executeTest("gzip -d ${strFileGz}");
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content');
#---------------------------------------------------------------------------------------------------------------------------
executeTest("gzip ${strFile}");
my $tFile = ${storageTest()->get($strFileGz)};
$oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip(
$oDriver->openWrite($strFile), {strCompressType => STORAGE_DECOMPRESS})}, '[object]', 'new write decompress');
$tBuffer = substr($tFile, 0, 10);
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, 10, ' write bytes');
$tBuffer = substr($tFile, 10);
$self->testResult(sub {$oGzipIo->write(\$tBuffer)}, length($tFile) - 10, ' write bytes');
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content');
}
################################################################################################################################
if ($self->begin('read()'))
{
my $tBuffer;
#---------------------------------------------------------------------------------------------------------------------------
my $oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openWrite($strFileGz), {bWantGzip => false})},
'[object]', 'new write compress');
$self->testResult(sub {$oGzipIo->write(\$strFileContent, $iFileLength)}, $iFileLength, ' write');
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
#---------------------------------------------------------------------------------------------------------------------------
$oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip(
$oDriver->openRead($strFileGz), {bWantGzip => false, strCompressType => STORAGE_DECOMPRESS})},
'[object]', 'new read decompress');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 4)}, 4, ' read 4 bytes');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 2, ' read 2 bytes');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 2, ' read 2 bytes');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes');
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
$self->testResult($tBuffer, $strFileContent, ' check content');
storageTest()->remove($strFileGz);
#---------------------------------------------------------------------------------------------------------------------------
$tBuffer = 'AA';
storageTest()->put($strFile, $strFileContent);
$oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile))},
'[object]', 'new read compress');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 10, ' read 10 bytes (request 2)');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 18, ' read 18 bytes (request 2)');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2)}, 0, ' read 0 bytes (request 2)');
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
$self->testResult(sub {storageTest()->put($strFileGz, substr($tBuffer, 2))}, 28, ' put content');
executeTest("gzip -df ${strFileGz}");
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check content');
#---------------------------------------------------------------------------------------------------------------------------
$tBuffer = undef;
executeTest('cat ' . $self->dataPath() . "/filecopy.archive2.bin | gzip -c > ${strFileGz}");
$oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip(
$oDriver->openRead($strFileGz), {lCompressBufferMax => 4096, strCompressType => STORAGE_DECOMPRESS})},
'[object]', 'new read decompress');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 8388608)}, 8388608, ' read 8388608 bytes');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 1)}, 0, ' read 0 bytes');
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
$self->testResult(sha1_hex($tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content');
storageTest()->remove($strFileGz);
#---------------------------------------------------------------------------------------------------------------------------
$tBuffer = undef;
executeTest('cp ' . $self->dataPath() . "/filecopy.archive2.bin ${strFile}");
$oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFile), {strCompressType => STORAGE_COMPRESS})},
'[object]', 'new read compress');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read bytes');
$self->testResult(sub {$oGzipIo->read(\$tBuffer, 2000000) > 0}, true, ' read bytes');
$self->testResult(sub {$oGzipIo->close()}, true, ' close');
$self->testResult(sub {storageTest()->put($strFileGz, $tBuffer) > 0}, true, ' put content');
executeTest("gzip -df ${strFileGz}");
$self->testResult(
sub {sha1_hex(${storageTest()->get($strFile)})}, '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content');
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strFileGz, $strFileContent);
$oGzipIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Gzip($oDriver->openRead($strFileGz), {strCompressType => STORAGE_DECOMPRESS})},
'[object]', 'new read decompress');
$self->testException(
sub {$oGzipIo->read(\$tBuffer, 1)}, ERROR_FILE_READ, "unable to inflate '${strFileGz}': incorrect header check");
}
}
1;
test/lib/pgBackRestTest/Module/Storage/StorageFilterShaTest.pm
+105
View File
@@ -0,0 +1,105 @@
####################################################################################################################################
# StorageFilterShaTest.pm - Tests for StorageFilterSha module.
####################################################################################################################################
package pgBackRestTest::Module::Storage::StorageFilterShaTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Digest::SHA qw(sha1_hex);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Filter::Sha;
use pgBackRest::Storage::Posix::Driver;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strFile = $self->testPath() . qw{/} . 'file.txt';
my $strFileContent = 'TESTDATA';
my $iFileLength = length($strFileContent);
my $oDriver = new pgBackRest::Storage::Posix::Driver();
################################################################################################################################
if ($self->begin('read()'))
{
my $tBuffer;
#---------------------------------------------------------------------------------------------------------------------------
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
my $oFileIo = $self->testResult(sub {$oDriver->openRead($strFile)}, '[object]', 'open read');
my $oShaIo = $self->testResult(sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new read');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, undef)}, 2, 'read 2 bytes');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 2)}, 2, 'read 2 bytes');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 4)}, 2, 'read 2 bytes');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 6)}, 2, 'read 2 bytes');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 2, 8)}, 0, 'read 0 bytes');
$self->testResult(sub {$oShaIo->close()}, true, 'close');
my $strSha = $self->testResult(
sub {$oShaIo->result(STORAGE_FILTER_SHA)}, sha1_hex($strFileContent), 'check hash against original content');
$self->testResult($strSha, sha1_hex($tBuffer), 'check hash against buffer');
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, 'check content');
#---------------------------------------------------------------------------------------------------------------------------
$tBuffer = undef;
$oFileIo = $self->testResult(
sub {$oDriver->openRead($self->dataPath() . '/filecopy.archive2.bin')}, '[object]', 'open read');
$oShaIo = $self->testResult(sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new read');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 8388608)}, 8388608, ' read 8388608 bytes');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 4194304)}, 4194304, ' read 4194304 bytes');
$self->testResult(sub {$oShaIo->read(\$tBuffer, 1)}, 0, ' read 0 bytes');
$self->testResult(sub {$oShaIo->close()}, true, ' close');
$self->testResult($oShaIo->result(STORAGE_FILTER_SHA), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check hash');
$self->testResult(sha1_hex($tBuffer), '1c7e00fd09b9dd11fc2966590b3e3274645dd031', ' check content');
}
################################################################################################################################
if ($self->begin('write()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oFileIo = $self->testResult(sub {$oDriver->openWrite($strFile, {bAtomic => true})}, '[object]', 'open write');
my $oShaIo = $self->testResult(
sub {new pgBackRest::Storage::Filter::Sha($oFileIo)}, '[object]', 'new');
my $tBuffer = substr($strFileContent, 0, 2);
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes');
$tBuffer = substr($strFileContent, 2, 2);
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes');
$tBuffer = substr($strFileContent, 4, 2);
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes');
$tBuffer = substr($strFileContent, 6, 2);
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 2, 'write 2 bytes');
$tBuffer = '';
$self->testResult(sub {$oShaIo->write(\$tBuffer)}, 0, 'write 0 bytes');
$self->testResult(sub {$oShaIo->close()}, true, 'close');
my $strSha = $self->testResult(
sub {$oShaIo->result(STORAGE_FILTER_SHA)}, sha1_hex($strFileContent), 'check hash against original content');
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, 'check content');
}
}
1;
test/lib/pgBackRestTest/Module/Storage/StorageHelperTest.pm
+147
View File
@@ -0,0 +1,147 @@
####################################################################################################################################
# StorageHelperTest.pm - Tests for Storage::Helper module.
####################################################################################################################################
package pgBackRestTest::Module::Storage::StorageHelperTest;
use parent 'pgBackRestTest::Env::ConfigEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Storable qw(dclone);
use pgBackRest::Config::Config;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# initTest - initialization before each test
####################################################################################################################################
sub initTest
{
my $self = shift;
storageTest()->pathCreate('db');
storageTest()->pathCreate('repo');
storageTest()->pathCreate('spool');
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Define test file
my $strFile = 'file.txt';
my $strFileCopy = 'file.txt.copy';
my $strFileContent = 'TESTDATA';
my $iFileSize = length($strFileContent);
# Setup parameters
my $oOption = {};
$self->optionSetTest($oOption, OPTION_DB_PATH, $self->testPath() . '/db');
$self->optionSetTest($oOption, OPTION_REPO_PATH, $self->testPath() . '/repo');
$self->optionSetTest($oOption, OPTION_SPOOL_PATH, $self->testPath() . '/spool');
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->optionBoolSetTest($oOption, OPTION_ARCHIVE_ASYNC, true);
#-------------------------------------------------------------------------------------------------------------------------------
if ($self->begin("storageLocal()"))
{
$self->testResult(sub {$self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH)}, '', 'config load');
$self->testResult(sub {storageLocal($self->testPath())->put($strFile, $strFileContent)}, $iFileSize, 'put');
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check put');
$self->testResult(sub {storageLocal($self->testPath())->put($strFile, $strFileContent)}, $iFileSize, 'put cache storage');
$self->testResult(sub {${storageTest()->get($strFile)}}, $strFileContent, ' check put');
}
#-------------------------------------------------------------------------------------------------------------------------------
if ($self->begin("storageDb()"))
{
$self->testResult(sub {$self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH)}, '', 'config load');
$self->testResult(sub {storageDb()->put($strFile, $strFileContent)}, $iFileSize, 'put');
$self->testResult(sub {${storageTest()->get("db/${strFile}")}}, $strFileContent, ' check put');
$self->testResult(sub {storageDb()->put($strFileCopy, $strFileContent)}, $iFileSize, 'put cached storage');
$self->testResult(sub {${storageTest()->get("db/${strFileCopy}")}}, $strFileContent, ' check put');
}
#-------------------------------------------------------------------------------------------------------------------------------
if ($self->begin("storageRepo()"))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH)}, '', 'config load');
$self->testResult(sub {storageRepo()->put($strFile, $strFileContent)}, $iFileSize, 'put');
$self->testResult(sub {${storageTest()->get("repo/${strFile}")}}, $strFileContent, ' check put');
$self->testResult(sub {storageRepo()->put($strFileCopy, $strFileContent)}, $iFileSize, 'put cached storage');
$self->testResult(sub {${storageTest()->get("repo/${strFileCopy}")}}, $strFileContent, ' check put');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE)}, $self->testPath() . '/repo/archive/db', 'check archive path');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . '/9.3-1/000000010000000100000001')},
$self->testPath() . '/repo/archive/db/9.3-1/0000000100000001/000000010000000100000001', 'check repo WAL file');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . '/9.3-1/0000000100000001')},
$self->testPath() . '/repo/archive/db/9.3-1/0000000100000001', 'check repo WAL path');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . '/9.3-1/0000000100000001')},
$self->testPath() . '/repo/archive/db/9.3-1/0000000100000001', 'check repo WAL major path');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . '/9.3-1')},
$self->testPath() . '/repo/archive/db/9.3-1', 'check repo archive id path');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_BACKUP)}, $self->testPath() . '/repo/backup/db', 'check backup path');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_BACKUP . '/file')}, $self->testPath() . '/repo/backup/db/file',
'check backup file');
#---------------------------------------------------------------------------------------------------------------------------
# Insert a bogus rule to generate an error
storageRepo()->{hRule}{'<BOGUS>'} =
{
fnRule => storageRepo()->{hRule}{&STORAGE_REPO_ARCHIVE}{fnRule},
};
$self->testException(sub {storageRepo()->pathGet('<BOGUS>')}, ERROR_ASSERT, 'invalid <REPO> storage rule <BOGUS>');
}
#-------------------------------------------------------------------------------------------------------------------------------
if ($self->begin("storageSpool()"))
{
$self->testResult(sub {$self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH)}, '', 'config load');
$self->testResult(sub {storageSpool()->put($strFile, $strFileContent)}, $iFileSize, 'put');
$self->testResult(sub {${storageTest()->get("spool/${strFile}")}}, $strFileContent, ' check put');
$self->testResult(sub {storageSpool()->put($strFileCopy, $strFileContent)}, $iFileSize, 'put cached storage');
$self->testResult(sub {${storageTest()->get("spool/${strFileCopy}")}}, $strFileContent, ' check put');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageSpool()->pathGet(STORAGE_SPOOL_ARCHIVE_OUT)}, $self->testPath() . '/spool/archive/db/out',
'check archive out path');
}
}
1;
test/lib/pgBackRestTest/Module/Storage/StorageLocalTest.pm
+363
View File
@@ -0,0 +1,363 @@
####################################################################################################################################
# StorageLocalTest.pm - Tests for Storage::Local module
####################################################################################################################################
package pgBackRestTest::Module::Storage::StorageLocalTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Digest::SHA qw(sha1_hex);
use pgBackRest::Config::Config;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Storage::Filter::Sha;
use pgBackRest::Storage::Local;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# initModule - common objects and variables used by all tests.
####################################################################################################################################
sub initModule
{
my $self = shift;
# Local path
$self->{strPathLocal} = $self->testPath() . '/local';
# Create the dynamic rule
my $fnRule = sub
{
my $strRule = shift;
my $strFile = shift;
my $xData = shift;
if ($strRule eq '<fn-rule-1>')
{
return "fn-rule-1/${xData}" . (defined($strFile) ? "/${strFile}" : '');
}
else
{
return 'fn-rule-2/' . (defined($strFile) ? "${strFile}/${strFile}" : 'no-file');
}
};
# Create the rule hash
my $hRule =
{
'<static-rule>' => 'static-rule-path',
'<fn-rule-1>' =>
{
fnRule => $fnRule,
xData => 'test',
},
'<fn-rule-2>' =>
{
fnRule => $fnRule,
},
};
# Create local storage
$self->{oStorageLocal} = new pgBackRest::Storage::Local(
$self->pathLocal(), new pgBackRest::Storage::Posix::Driver(), {hRule => $hRule, bAllowTemp => false});
# Remote path
$self->{strPathRemote} = $self->testPath() . '/remote';
# Create the repo path so the remote won't complain that it's missing
mkdir($self->pathRemote())
or confess &log(ERROR, "unable to create repo directory '" . $self->pathRemote() . qw{'});
# Remove repo path now that the remote is created
rmdir($self->{strPathRemote})
or confess &log(ERROR, "unable to remove repo directory '" . $self->pathRemote() . qw{'});
# Create remote storage
$self->{oStorageRemote} = new pgBackRest::Storage::Local(
$self->pathRemote(), new pgBackRest::Storage::Posix::Driver(), {hRule => $hRule});
}
####################################################################################################################################
# initTest - initialization before each test
####################################################################################################################################
sub initTest
{
my $self = shift;
executeTest(
'ssh ' . $self->backrestUser() . '\@' . $self->host() . ' mkdir -m 700 ' . $self->pathRemote(), {bSuppressStdErr => true});
executeTest('mkdir -m 700 ' . $self->pathLocal());
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Define test file
my $strFile = 'file.txt';
my $strFileCopy = 'file.txt.copy';
# my $strFileHash = 'bbbcf2c59433f68f22376cd2439d6cd309378df6';
my $strFileContent = 'TESTDATA';
my $iFileSize = length($strFileContent);
#---------------------------------------------------------------------------------------------------------------------------
if ($self->begin("pathGet()"))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->pathGet('<static-rule>/test', {bTemp => true})},
ERROR_ASSERT, "temp file not supported for storage '" . $self->storageLocal()->pathBase() . "'");
$self->testException(
sub {$self->storageRemote()->pathGet('<static-rule>', {bTemp => true})},
ERROR_ASSERT, 'file part must be defined when temp file specified');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageRemote()->pathGet('/file', {bTemp => true})}, "/file.tmp", 'absolute path temp');
$self->testResult(sub {$self->storageRemote()->pathGet('/file')}, "/file", 'absolute path file');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->pathGet('file')}, $self->storageLocal()->pathBase() . '/file', 'relative path');
$self->testResult(
sub {$self->storageRemote()->pathGet('file', {bTemp => true})},
$self->storageRemote()->pathBase() . '/file.tmp', 'relative path temp');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->pathGet('<static-rule/file')}, ERROR_ASSERT, "found < but not > in '<static-rule/file'");
$self->testException(
sub {$self->storageLocal()->pathGet('<bogus-rule>')}, ERROR_ASSERT, "storage rule '<bogus-rule>' does not exist");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->pathGet('<static-rule>/file')},
$self->storageLocal()->pathBase() . '/static-rule-path/file', 'static rule file');
$self->testResult(
sub {$self->storageLocal()->pathGet('<static-rule>')},
$self->storageLocal()->pathBase() . '/static-rule-path', 'static rule path');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->pathGet('<fn-rule-1>/file')},
$self->storageLocal()->pathBase() . '/fn-rule-1/test/file', 'function rule 1 file');
$self->testResult(
sub {$self->storageLocal()->pathGet('<fn-rule-2>/file')},
$self->storageLocal()->pathBase() . '/fn-rule-2/file/file', 'function rule 2 file');
$self->testResult(
sub {$self->storageLocal()->pathGet('<fn-rule-1>')},
$self->storageLocal()->pathBase() . '/fn-rule-1/test', 'function rule 1 path');
$self->testResult(
sub {$self->storageLocal()->pathGet('<fn-rule-2>')},
$self->storageLocal()->pathBase() . '/fn-rule-2/no-file', 'function rule 2 no file');
}
################################################################################################################################
if ($self->begin('openWrite()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oFileIo = $self->testResult(sub {$self->storageLocal()->openWrite($strFile)}, '[object]', 'open write');
$self->testResult(sub {$oFileIo->write(\$strFileContent, length($strFileContent))}, $iFileSize, "write $iFileSize bytes");
$self->testResult(sub {$oFileIo->close()}, true, 'close');
}
################################################################################################################################
if ($self->begin('put()'))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile))}, 0, 'put empty');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($strFile)}, 0, 'put empty (all defaults)');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent)}, $iFileSize, 'put');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), \$strFileContent)}, $iFileSize,
'put reference');
}
################################################################################################################################
if ($self->begin('openRead()'))
{
my $tContent;
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->openRead($strFile, {bIgnoreMissing => true})}, undef, 'ignore missing');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->openRead($strFile)}, ERROR_FILE_MISSING,
"unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory");
#---------------------------------------------------------------------------------------------------------------------------
executeTest('sudo touch ' . $self->pathLocal() . "/${strFile} && sudo chmod 700 " . $self->pathLocal() . "/${strFile}");
$self->testException(
sub {$self->storageLocal()->openRead($strFile)}, ERROR_FILE_OPEN,
"unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': Permission denied");
executeTest('sudo rm ' . $self->pathLocal() . "/${strFile}");
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent);
my $oFileIo = $self->testResult(sub {$self->storageLocal()->openRead($strFile)}, '[object]', 'open read');
$self->testResult(sub {$oFileIo->read(\$tContent, $iFileSize)}, $iFileSize, "read $iFileSize bytes");
$self->testResult($tContent, $strFileContent, ' check read');
#---------------------------------------------------------------------------------------------------------------------------
$oFileIo = $self->testResult(
sub {$self->storageLocal()->openRead($strFile, {rhyFilter => [{strClass => STORAGE_FILTER_SHA}]})}, '[object]',
'open read + checksum');
undef($tContent);
$self->testResult(sub {$oFileIo->read(\$tContent, $iFileSize)}, $iFileSize, "read $iFileSize bytes");
$self->testResult(sub {$oFileIo->close()}, true, 'close');
$self->testResult($tContent, $strFileContent, ' check read');
$self->testResult($oFileIo->result(STORAGE_FILTER_SHA), sha1_hex($strFileContent), ' check hash');
}
################################################################################################################################
if ($self->begin('get()'))
{
my $tBuffer;
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->get($self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}))}, undef,
'get missing');
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->put($strFile);
$self->testResult(sub {${$self->storageLocal()->get($strFile)}}, undef, 'get empty');
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->put($strFile, $strFileContent);
$self->testResult(sub {${$self->storageLocal()->get($strFile)}}, $strFileContent, 'get');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {${$self->storageLocal()->get($self->storageLocal()->openRead($strFile))}}, $strFileContent, 'get from io');
}
################################################################################################################################
if ($self->begin('hashSize()'))
{
my $tBuffer;
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($strFile, $strFileContent)}, 8, 'put');
$self->testResult(
sub {$self->storageLocal()->hashSize($strFile)},
qw{(} . sha1_hex($strFileContent) . ', ' . $iFileSize . qw{)}, ' check hash/size');
}
################################################################################################################################
if ($self->begin('copy()'))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING,
"unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory");
$self->testResult(
sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->copy(
$self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}),
$self->storageLocal()->openWrite($strFileCopy))},
false, 'missing source io');
$self->testResult(
sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING,
"unable to open '" . $self->storageLocal()->pathBase() . "/${strFile}': No such file or directory");
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->put($strFile, $strFileContent);
$self->testResult(sub {$self->storageLocal()->copy($strFile, $strFileCopy)}, true, 'copy filename->filename');
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->remove($strFileCopy);
$self->testResult(
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, true, 'copy io->filename');
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->remove($strFileCopy);
$self->testResult(
sub {$self->storageLocal()->copy(
$self->storageLocal()->openRead($strFile), $self->storageLocal()->openWrite($strFileCopy))},
true, 'copy io->io');
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
}
################################################################################################################################
if ($self->begin('info()'))
{
$self->testResult(sub {$self->storageLocal()->info($self->{strPathLocal})}, "[object]", 'stat dir successfully');
$self->testException(sub {$self->storageLocal()->info($strFile)}, ERROR_FILE_MISSING,
"unable to stat '". $self->{strPathLocal} . "/" . $strFile ."': No such file or directory");
}
################################################################################################################################
if ($self->begin('pathCreate()'))
{
my $strTestPath = $self->{strPathLocal} . "/" . BOGUS;
$self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath)}, "[undef]",
"test creation of path " . $strTestPath);
$self->testException(sub {$self->storageLocal()->pathCreate($strTestPath)}, ERROR_PATH_EXISTS,
"unable to create path '". $strTestPath. "' because it already exists");
$self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath, {bIgnoreExists => true})}, "[undef]",
"ignore path exists");
}
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub host {return '127.0.0.1'}
sub pathLocal {return shift->{strPathLocal}};
sub pathRemote {return shift->{strPathRemote}};
sub protocolLocal {return shift->{oProtocolLocal}};
sub protocolRemote {return shift->{oProtocolRemote}};
sub storageLocal {return shift->{oStorageLocal}};
sub storageRemote {return shift->{oStorageRemote}};
1;
test/lib/pgBackRestTest/Module/Storage/StoragePosixTest.pm
+389
View File
@@ -0,0 +1,389 @@
####################################################################################################################################
# Posix Driver Tests
####################################################################################################################################
package pgBackRestTest::Module::Storage::StoragePosixTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(basename dirname);
use IO::Socket::UNIX;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Storage::Posix::Driver;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strFile = $self->testPath() . '/file.txt';
my $strFileContent = 'TESTDATA';
my $iFileLength = length($strFileContent);
my $iFileLengthHalf = int($iFileLength / 2);
# Test driver
my $oPosix = new pgBackRest::Storage::Posix::Driver();
################################################################################################################################
if ($self->begin('exists()'))
{
my $strPathSub = $self->testPath() . '/sub';
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oPosix->exists($strFile)}, false, 'file');
#---------------------------------------------------------------------------------------------------------------------------
executeTest("sudo mkdir ${strPathSub} && sudo chmod 700 ${strPathSub}");
$self->testResult(
sub {$oPosix->pathExists($strPathSub)}, true, 'path');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$oPosix->exists("${strPathSub}/file")}, ERROR_FILE_EXISTS,
"unable to test if file '${strPathSub}/file' exists: Permission denied");
}
################################################################################################################################
if ($self->begin("manifestList()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my @stryFile = ('.', 'test.txt');
$self->testResult(
sub {$oPosix->manifestList($self->testPath(), \@stryFile)},
'{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}}',
'skip missing file');
}
################################################################################################################################
if ($self->begin("manifestStat()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strFile = $self->testPath() . '/test.txt';
$self->testResult(sub {$oPosix->manifestStat($strFile)}, '[undef]', 'ignore missing file');
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strFile, "TEST");
utime(1111111111, 1111111111, $strFile);
executeTest('chmod 1640 ' . $strFile);
$self->testResult(
sub {$oPosix->manifestStat($strFile)},
'{group => ' . $self->group() .
', mode => 1640, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}',
'stat file');
#---------------------------------------------------------------------------------------------------------------------------
my $strSocketFile = $self->testPath() . '/test.socket';
# Create a socket to test invalid files
my $oSocket = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Local => $strSocketFile, Listen => 1);
$self->testException(
sub {$oPosix->manifestStat($strSocketFile)}, ERROR_FILE_INVALID,
"${strSocketFile} is not of type directory, file, or link");
# Cleanup socket
$oSocket->close();
storageTest()->remove($strSocketFile);
#---------------------------------------------------------------------------------------------------------------------------
my $strTestPath = $self->testPath() . '/public_dir';
storageTest()->pathCreate($strTestPath, {strMode => '0750'});
$self->testResult(
sub {$oPosix->manifestStat($strTestPath)},
'{group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}',
'stat directory');
#---------------------------------------------------------------------------------------------------------------------------
my $strTestLink = $self->testPath() . '/public_dir_link';
symlink($strTestPath, $strTestLink)
or confess &log(ERROR, "unable to create symlink from ${strTestPath} to ${strTestLink}");
$self->testResult(
sub {$oPosix->manifestStat($strTestLink)},
'{group => ' . $self->group() . ", link_destination => ${strTestPath}, type => l, user => " . $self->pgUser() . '}',
'stat link');
}
################################################################################################################################
if ($self->begin("manifestRecurse()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strTestPath = $self->testPath() . '/public_dir';
my $strTestFile = "${strTestPath}/test.txt";
$self->testException(
sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestFile, undef, 0, $hManifest); $hManifest},
ERROR_FILE_MISSING, "unable to stat '${strTestFile}': No such file or directory");
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->pathCreate($strTestPath, {strMode => '0750'});
$self->testResult(
sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestPath, undef, 0, $hManifest); $hManifest},
'{. => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}}',
'empty directory manifest');
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strTestFile, "TEST");
utime(1111111111, 1111111111, $strTestFile);
executeTest('chmod 0750 ' . $strTestFile);
storageTest()->pathCreate("${strTestPath}/sub", {strMode => '0750'});
$self->testResult(
sub {my $hManifest = {}; $oPosix->manifestRecurse(
$self->testPath(), basename($strTestPath), 1, $hManifest); $hManifest},
'{public_dir => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'public_dir/sub => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'public_dir/' . basename($strTestFile) . ' => {group => ' . $self->group() .
', mode => 0750, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}}',
'directory and file manifest');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {my $hManifest = {}; $oPosix->manifestRecurse($strTestFile, undef, 0, $hManifest); $hManifest},
'{' . basename($strTestFile) . ' => {group => ' . $self->group() .
', mode => 0750, modification_time => 1111111111, size => 4, type => f, user => ' . $self->pgUser() . '}}',
'single file manifest');
}
################################################################################################################################
if ($self->begin("manifest()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strMissingFile = $self->testPath() . '/missing';
$self->testException(
sub {$oPosix->manifest($strMissingFile)},
ERROR_FILE_MISSING, "unable to stat '${strMissingFile}': No such file or directory");
#---------------------------------------------------------------------------------------------------------------------------
# Setup test data
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1');
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1/sub2');
executeTest("echo 'TESTDATA' > " . $self->testPath() . '/test.txt');
utime(1111111111, 1111111111, $self->testPath() . '/test.txt');
executeTest('chmod 1640 ' . $self->testPath() . '/test.txt');
executeTest("echo 'TESTDATA_' > ". $self->testPath() . '/sub1/test-sub1.txt');
utime(1111111112, 1111111112, $self->testPath() . '/sub1/test-sub1.txt');
executeTest('chmod 0640 ' . $self->testPath() . '/sub1/test-sub1.txt');
executeTest("echo 'TESTDATA__' > " . $self->testPath() . '/sub1/sub2/test-sub2.txt');
utime(1111111113, 1111111113, $self->testPath() . '/sub1/sub2/test-sub2.txt');
executeTest('chmod 0646 ' . $self->testPath() . '/sub1/test-sub1.txt');
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/test-hardlink.txt');
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/sub2/test-hardlink.txt');
executeTest('ln -s .. ' . $self->testPath() . '/sub1/test');
executeTest('chmod 0700 ' . $self->testPath() . '/sub1/test');
executeTest('ln -s ../.. ' . $self->testPath() . '/sub1/sub2/test');
executeTest('chmod 0750 ' . $self->testPath() . '/sub1/sub2/test');
executeTest('chmod 0770 ' . $self->testPath());
$self->testResult(
sub {$oPosix->manifest($self->testPath())},
'{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1/sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1/sub2/test => {group => ' . $self->group() . ', link_destination => ../.., type => l, user => ' .
$self->pgUser() . '}, ' .
'sub1/sub2/test-hardlink.txt => ' .
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/sub2/test-sub2.txt => ' .
'{group => ' . $self->group() . ', mode => 0666, modification_time => 1111111113, size => 11, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/test => {group => ' . $self->group() . ', link_destination => .., type => l, user => ' . $self->pgUser() . '}, ' .
'sub1/test-hardlink.txt => ' .
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/test-sub1.txt => ' .
'{group => ' . $self->group() . ', mode => 0646, modification_time => 1111111112, size => 10, type => f, user => ' .
$self->pgUser() . '}, ' .
'test.txt => ' .
'{group => ' . $self->group() . ', mode => 1640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}}',
'complete manifest');
}
################################################################################################################################
if ($self->begin('openRead() & Posix::FileRead'))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$oPosix->openRead($strFile)}, ERROR_FILE_MISSING, "unable to open '${strFile}': No such file or directory");
#---------------------------------------------------------------------------------------------------------------------------
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
$self->testResult(
sub {$oPosix->openRead($strFile)}, '[object]', 'open read');
}
################################################################################################################################
if ($self->begin('openWrite() & Posix::FileWrite'))
{
my $tContent = $strFileContent;
#---------------------------------------------------------------------------------------------------------------------------
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
executeTest("chmod 600 ${strFile} && sudo chown root:root ${strFile}");
$self->testException(
sub {new pgBackRest::Storage::Posix::FileRead($oPosix, $strFile)}, ERROR_FILE_OPEN,
"unable to open '${strFile}': Permission denied");
executeTest("sudo rm -rf ${strFile}");
#---------------------------------------------------------------------------------------------------------------------------
my $oPosixIo = $self->testResult(
sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile)}, '[object]', 'open');
$tContent = substr($strFileContent, 0, $iFileLengthHalf);
$self->testResult(
sub {$oPosixIo->write(\$tContent)}, $iFileLengthHalf, 'write part 1');
$tContent = substr($strFileContent, $iFileLengthHalf);
$self->testResult(
sub {$oPosixIo->write(\$tContent)}, $iFileLength - $iFileLengthHalf,
'write part 2');
$oPosixIo->close();
$tContent = undef;
$self->testResult(
sub {(new pgBackRest::Storage::Posix::FileRead($oPosix, $strFile))->read(\$tContent, $iFileLength)},
$iFileLength, 'check write content length');
$self->testResult($tContent, $strFileContent, 'check write content');
#---------------------------------------------------------------------------------------------------------------------------
$oPosixIo = $self->testResult(
sub {new pgBackRest::Storage::Posix::FileWrite(
$oPosix, "${strFile}.atomic", {bAtomic => true, strMode => '0666', lTimestamp => time(), bSync => false})},
'[object]', 'open');
$self->testResult(sub {$oPosixIo->write(\$tContent, $iFileLength)}, $iFileLength, 'write');
$self->testResult(sub {$oPosixIo->close()}, true, 'close');
$self->testResult(sub {${storageTest()->get("${strFile}.atomic")}}, $strFileContent, 'check content');
#---------------------------------------------------------------------------------------------------------------------------
$oPosixIo = $self->testResult(
sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile)}, '[object]', 'open');
$self->testResult(sub {$oPosixIo->close()}, true, 'close');
undef($oPosixIo);
#---------------------------------------------------------------------------------------------------------------------------
$oPosixIo = $self->testResult(
sub {new pgBackRest::Storage::Posix::FileWrite($oPosix, $strFile, {lTimestamp => time()})}, '[object]', 'open');
$self->testResult(sub {$oPosixIo->write(\$strFileContent, $iFileLength)}, $iFileLength, 'write');
executeTest("rm -f $strFile");
$self->testException(
sub {$oPosixIo->close()}, ERROR_FILE_WRITE, "unable to set time for '${strFile}': No such file or directory");
}
################################################################################################################################
if ($self->begin('pathCreate()'))
{
my $strPathParent = $self->testPath() . '/parent';
my $strPathSub = "${strPathParent}/sub1/sub2";
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oPosix->pathCreate($strPathParent)}, undef, 'parent path');
$self->testResult(
sub {$oPosix->pathExists($strPathParent)}, true, ' check path');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$oPosix->pathCreate($strPathParent)}, ERROR_PATH_EXISTS,
"unable to create path '${strPathParent}' because it already exists");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oPosix->pathCreate($strPathParent, {bIgnoreExists => true})}, undef, 'path already exists');
#---------------------------------------------------------------------------------------------------------------------------
executeTest("sudo chown root:root ${strPathParent} && sudo chmod 700 ${strPathParent}");
$self->testException(
sub {$oPosix->pathCreate($strPathSub)}, ERROR_PATH_CREATE,
"unable to create path '${strPathSub}': Permission denied");
#---------------------------------------------------------------------------------------------------------------------------
executeTest("rmdir ${strPathParent}");
$self->testException(
sub {$oPosix->pathCreate($strPathSub)}, ERROR_PATH_MISSING,
"unable to create path '${strPathSub}' because parent does not exist");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oPosix->pathCreate($strPathSub, {bCreateParent => true})}, undef, 'path with parents');
$self->testResult(
sub {$oPosix->pathExists($strPathSub)}, true, ' check path');
}
################################################################################################################################
if ($self->begin('move()'))
{
my $strFileCopy = "${strFile}.copy";
my $strFileSub = $self->testPath() . '/sub/file.txt';
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$oPosix->move($strFile, $strFileCopy)}, ERROR_FILE_MISSING,
"unable to move '${strFile}' because it is missing");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {storageTest()->put($strFile, $strFileContent)}, $iFileLength, 'put');
$self->testResult(
sub {$oPosix->move($strFile, $strFileCopy)}, undef, 'simple move');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$oPosix->move($strFileCopy, $strFileSub)}, ERROR_PATH_MISSING,
"unable to move '${strFileCopy}' to missing path '" . dirname($strFileSub) . "'");
#---------------------------------------------------------------------------------------------------------------------------
executeTest('sudo mkdir ' . dirname($strFileSub) . ' && sudo chmod 700 ' . dirname($strFileSub));
$self->testException(
sub {$oPosix->move($strFileCopy, $strFileSub)}, ERROR_FILE_MOVE,
"unable to move '${strFileCopy}' to '${strFileSub}': Permission denied");
executeTest('sudo rmdir ' . dirname($strFileSub));
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oPosix->move($strFileCopy, $strFileSub, {bCreatePath => true})}, undef, 'create parent path');
}
}
1;