db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Remove LibC.

Browse Source 
This was the interface between Perl and C introduced in 36a5349b but since f0ef73db has only been used by the Perl integration tests.  This is expensive code to maintain just for testing.

The main dependency was the interface to storage, no matter where it was located, e.g. S3.  Replace this with the new-introduced repo commands (d3c83453) that allow access to repo storage via the command line.

The other dependency was on various cfgOption* functions and CFGOPT_ constants that were convenient but not necessary.  Replace these with hard-coded strings in most places and create new constants for commonly used values.

Remove all auto-generated Perl code.  This means that the error list will no longer be maintained automatically so copy used errors to Common::Exception.pm.  This file will need to be maintained manually going forward but there is not likely to be much churn as the Perl integration tests are being retired.

Update test.pl and related code to remove LibC builds.

Ding, dong, LibC is dead.
This commit is contained in:
David Steele
2020-03-09 17:41:59 -04:00
parent d3c83453de
commit 79cfd3aebf
62 changed files with 1203 additions and 4643 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/lib/pgBackRestTest/Common/BuildTest.pm
+2 -25
View File
@@ -113,7 +113,7 @@ sub buildDependencyTree
# Only process non-auto files
if ($strFile =~ /^[A-Za-z0-9\/]+\.(c|h)$/)
{
buildDependencyTreeSub($oStorage, $rhDependencyTree, $strFile, true, undef, ['src', 'libc']);
buildDependencyTreeSub($oStorage, $rhDependencyTree, $strFile, true, undef, ['src']);
}
}
@@ -229,8 +229,7 @@ sub buildMakefileObjectCompile
foreach my $strInclude (@{$rhDependencyTree->{$strFile}{include}})
{
$strDepend .=
' ' . ($oStorage->exists("src/${strInclude}") || $strInclude eq BUILD_AUTO_H ? '' : '../libc/') . $strInclude;
$strDepend .= " ${strInclude}";
}
$strMakefile .=
@@ -288,27 +287,5 @@ sub buildMakefile
push @EXPORT, qw(buildMakefile);
####################################################################################################################################
# Load the C library and check pointer size
####################################################################################################################################
sub buildLoadLibC
{
# Load the module dynamically
require pgBackRest::LibC;
pgBackRest::LibC->import(qw(:debug));
# Load shared object
require XSLoader;
XSLoader::load('pgBackRest::LibC', '999');
# Do a basic test to make sure it installed correctly
if (libcUvSize() != 8)
{
confess &log(ERROR, 'UVSIZE in test library does not equal 8');
}
}
push @EXPORT, qw(buildLoadLibC);
1;
test/lib/pgBackRestTest/Common/CodeCountTest.pm
+3 -5
View File
@@ -55,7 +55,6 @@ sub codeCountScan
$strFile =~ '\.eps$' ||
$strFile =~ '\.cache$' ||
$strFile =~ '^doc/site/' ||
$strFile eq 'libc/typemap' ||
$strFile eq 'test/Vagrantfile' ||
$strFile =~ '^test/\.vagrant/' ||
$strFile =~ '^test/certificate/' ||
@@ -83,13 +82,12 @@ sub codeCountScan
{
$strClass = 'doc/core';
}
elsif ($strFile =~ '^build/' || $strFile =~ '^libc/build/' || $strFile eq 'libc/Makefile.PL' ||
$strFile eq 'src/Makefile.in' || $strFile eq 'src/configure' || $strFile eq 'src/configure.ac')
elsif ($strFile =~ '^build/' || $strFile eq 'src/Makefile.in' || $strFile eq 'src/configure' ||
$strFile eq 'src/configure.ac')
{
$strClass = 'build';
}
elsif ($strFile =~ '^test/lib/pgBackRestTest/Module/' || $strFile =~ '^test/src/module/' ||
$strFile =~ '^libc/t/')
elsif ($strFile =~ '^test/lib/pgBackRestTest/Module/' || $strFile =~ '^test/src/module/')
{
$strClass = 'test/module';
}
test/lib/pgBackRestTest/Common/FileTest.pm
+25 -77
View File
@@ -25,7 +25,6 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Storage::Base;
@@ -154,7 +153,7 @@ sub forceStorageMode
(
$strOperation,
$oStorage,
$strPathExp,
$strPath,
$strMode,
$bRecurse
) =
@@ -162,15 +161,15 @@ sub forceStorageMode
(
__PACKAGE__ . '::forceStorageMode', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'strPath'},
{name => 'strMode'},
{name => 'bRecurse', optional => true, default => false},
);
# Mode commands are ignored on S3
if (!$oStorage->can('type') || $oStorage->type() ne STORAGE_S3)
# Mode commands are ignored on object storage
if ($oStorage->type() ne STORAGE_OBJECT)
{
executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp));
executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} ${strPath}");
}
# Return from function and log return values if any
@@ -189,55 +188,48 @@ sub forceStorageMove
(
$strOperation,
$oStorage,
$strSourcePathExp,
$strDestinationPathExp,
$strSourcePath,
$strDestinationPath,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->forceStorageMove', \@_,
{name => 'oStorage'},
{name => 'strSourcePathExp'},
{name => 'strDestinationPathExp'},
{name => 'strSourcePath'},
{name => 'strDestinationPath'},
{name => 'bRecurse', optional => true, default => true},
);
# If S3 then use storage commands to remove
if ($oStorage->type() eq STORAGE_S3)
# If object storage then use storage commands to remove
if ($oStorage->type() eq STORAGE_OBJECT)
{
if ($bRecurse)
{
my $rhManifest = $oStorage->manifest($strSourcePathExp);
my $rhManifest = $oStorage->manifest($strSourcePath);
foreach my $strName (sort(keys(%{$rhManifest})))
{
if ($rhManifest->{$strName}{type} eq 'f')
{
$oStorage->put(
new pgBackRest::Storage::StorageWrite(
$oStorage,
pgBackRest::LibC::StorageWrite->new(
$oStorage->{oStorageC}, "${strDestinationPathExp}/${strName}", 0, undef, undef, 0, true, false)),
${$oStorage->get(
new pgBackRest::Storage::StorageRead(
$oStorage,
pgBackRest::LibC::StorageRead->new(
$oStorage->{oStorageC}, "${strSourcePathExp}/${strName}", false)))});
$oStorage->remove("${strSourcePathExp}/${strName}");
"${strDestinationPath}/${strName}", ${$oStorage->get("${strSourcePath}/${strName}", {bRaw => true})},
{bRaw => true});
}
}
$oStorage->pathRemove($strSourcePath, {bRecurse => true});
}
else
{
$oStorage->put($strDestinationPathExp, ${$oStorage->get($strSourcePathExp)});
$oStorage->remove($strSourcePathExp);
$oStorage->put($strDestinationPath, ${$oStorage->get($strSourcePath)});
$oStorage->remove($strSourcePath);
}
}
# Else remove using filesystem commands
else
{
executeTest('mv ' . $oStorage->pathGet($strSourcePathExp) . ' ' . $oStorage->pathGet($strDestinationPathExp));
executeTest("mv ${strSourcePath} ${strDestinationPath}");
}
# Return from function and log return values if any
@@ -246,41 +238,6 @@ sub forceStorageMove
push(@EXPORT, qw(forceStorageMove));
####################################################################################################################################
# forceStorageOwner - force ownership on a file or path
####################################################################################################################################
sub forceStorageOwner
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strPathExp,
$strOwner,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '::forceStorageOwner', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'strOwner'},
{name => 'bRecurse', optional => true, default => false},
);
# Owner commands are ignored on S3
if ($oStorage->type() ne STORAGE_S3)
{
executeTest('chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp));
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageOwner));
####################################################################################################################################
# forceStorageRemove - force remove a file or path from storage
####################################################################################################################################
@@ -291,34 +248,25 @@ sub forceStorageRemove
(
$strOperation,
$oStorage,
$strPathExp,
$strPath,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '->forceStorageRemove', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'strPath'},
{name => 'bRecurse', optional => true, default => false},
);
# If S3 then use storage commands to remove
if ($oStorage->type() eq STORAGE_S3)
# If object storage then use storage commands to remove
if ($oStorage->type() eq STORAGE_OBJECT)
{
my $oInfo = $oStorage->info($strPathExp, {bIgnoreMissing => true});
if (defined($oInfo) && $oInfo->{type} eq 'f')
{
$oStorage->remove($strPathExp);
}
else
{
$oStorage->pathRemove($strPathExp, {bRecurse => true});
}
$oStorage->pathRemove($strPath, {bRecurse => true});
}
else
{
executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp));
executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $strPath);
}
# Return from function and log return values if any
test/lib/pgBackRestTest/Common/JobTest.pm
+1 -4
View File
@@ -221,13 +221,11 @@ sub run
if (!$rhBuildInit->{$self->{oTest}->{&TEST_VM}}{$self->{iVmIdx}})
{
executeTest(
'rsync -rt --delete --exclude=*.o --exclude=test.c --exclude=test.gcno --exclude=LibC.h --exclude=xs' .
'rsync -rt --delete --exclude=*.o --exclude=test.c --exclude=test.gcno ' .
' --exclude=test --exclude=buildflags --exclude=testflags --exclude=harnessflags' .
' --exclude=build.auto.h --exclude=build.auto.h.in --exclude=Makefile --exclude=Makefile.in' .
' --exclude=configure --exclude=configure.ac' .
" $self->{strBackRestBase}/src/ $self->{strGCovPath} && " .
"rsync -t $self->{strBackRestBase}/libc/LibC.h $self->{strGCovPath} && " .
"rsync -rt --delete $self->{strBackRestBase}/libc/xs/ $self->{strGCovPath}/xs && " .
"rsync -rt --delete --exclude=*.o $self->{strBackRestBase}/test/src/ $self->{strGCovPath}/test");
}
@@ -794,7 +792,6 @@ sub jobInstallC
my $oVm = vmGet();
my $strBuildPath = "${strBasePath}/test/.vagrant/bin/${strVm}";
my $strBuildLibCPath = "${strBuildPath}/libc";
my $strBuildBinPath = "${strBuildPath}/src";
executeTest(
test/lib/pgBackRestTest/Common/ListTest.pm
+1 -1
View File
@@ -115,7 +115,7 @@ sub testListGet
# Skip this test if it is integration and vm=none
next if ($strVm eq VM_NONE && $hTest->{&TESTDEF_TYPE} eq TESTDEF_INTEGRATION);
# Skip this test if it is not C and vm=none. Perl tests require libc which is not supported.
# Skip this test if it is not C and vm=none. Perl tests require Docker which is not supported.
next if ($strVm eq VM_NONE && !$hTest->{&TESTDEF_C});
# Skip this test if a container is required and vm=none.
test/lib/pgBackRestTest/Common/RunTest.pm
+3 -5
View File
@@ -114,7 +114,7 @@ sub process
$self->{iVmId},
$self->{strBasePath},
$self->{strTestPath},
$self->{strBackRestExeC},
$self->{strBackRestExe},
$self->{strBackRestExeHelper},
$self->{strPgBinPath},
$self->{strPgVersion},
@@ -136,7 +136,7 @@ sub process
{name => 'iVmId'},
{name => 'strBasePath'},
{name => 'strTestPath'},
{name => 'strBackRestExeC'},
{name => 'strBackRestExe'},
{name => 'strBackRestExeHelper'},
{name => 'strPgBinPath', required => false},
{name => 'strPgVersion', required => false},
@@ -159,9 +159,6 @@ sub process
$oStorage = new pgBackRestTest::Common::Storage(
$self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false}));
# Generate backrest exe
$self->{strBackRestExe} = defined($self->{strBackRestExeC}) ? $self->{strBackRestExeC} : $self->{strBackRestExeHelper};
projectBinSet($self->{strBackRestExe});
# Init, run, and end the test(s)
@@ -545,6 +542,7 @@ push(@EXPORT, qw(storageTest));
####################################################################################################################################
sub archBits {return vmArchBits(shift->{strVm})}
sub backrestExe {return shift->{strBackRestExe}}
sub backrestExeHelper {return shift->{strBackRestExeHelper}}
sub basePath {return shift->{strBasePath}}
sub dataPath {return shift->basePath() . '/test/data'}
sub doCleanup {return shift->{bCleanup}}
test/lib/pgBackRestTest/Common/Storage.pm
+186 -2
View File
@@ -69,6 +69,85 @@ sub new
);
}
####################################################################################################################################
# Copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed instead of
# file names.
####################################################################################################################################
sub copy
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xSourceFile,
$xDestinationFile,
$bSourceOpen,
) =
logDebugParam
(
__PACKAGE__ . '->copy', \@_,
{name => 'xSourceFile', required => false},
{name => 'xDestinationFile'},
{name => 'bSourceOpen', optional => true, default => false},
);
# Is source/destination an IO object or a file expression?
my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef;
# Does the source file exist?
my $bResult = false;
# Copy if the source file exists
if (defined($oSourceFileIo))
{
my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile);
# Use C copy if source and destination are C objects
if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite}))
{
$bResult = $self->{oStorageC}->copy(
$oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false;
}
else
{
# Open the source file if it is a C object
$bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true;
if ($bResult)
{
# Open the destination file if it is a C object
if (defined($oDestinationFileIo->{oStorageCWrite}))
{
$oDestinationFileIo->open();
}
# Copy the data
do
{
# Read data
my $tBuffer = '';
$oSourceFileIo->read(\$tBuffer, $self->{lBufferMax});
$oDestinationFileIo->write(\$tBuffer);
}
while (!$oSourceFileIo->eof());
# Close files
$oSourceFileIo->close();
$oDestinationFileIo->close();
}
}
}
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => $bResult, trace => true},
);
}
####################################################################################################################################
# exists - check if file exists
####################################################################################################################################
@@ -99,6 +178,62 @@ sub exists
);
}
####################################################################################################################################
# get - reads a buffer from storage all at once
####################################################################################################################################
sub get
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFile,
) =
logDebugParam
(
__PACKAGE__ . '->get', \@_,
{name => 'xFile', required => false, trace => true},
);
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or
# if the repo has a user passphrase defined - else pass undef
my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead($xFile)) : undef;
# Read only if there is something to read from
my $tContent;
my $lSize = 0;
if (defined($oFileIo))
{
my $lSizeRead;
do
{
$lSizeRead = $oFileIo->read(\$tContent, $self->{lBufferMax});
$lSize += $lSizeRead;
}
while ($lSizeRead != 0);
# Close the file
$oFileIo->close();
# If nothing was read then set to undef
if ($lSize == 0)
{
$tContent = undef;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rtContent', value => defined($oFileIo) ? \$tContent : undef, trace => true},
);
}
####################################################################################################################################
# info - get information for path/file
####################################################################################################################################
@@ -595,6 +730,56 @@ sub pathSync
return logDebugReturn($strOperation);
}
####################################################################################################################################
# put - writes a buffer out to storage all at once
####################################################################################################################################
sub put
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFile,
$xContent,
) =
logDebugParam
(
__PACKAGE__ . '->put', \@_,
{name => 'xFile', trace => true},
{name => 'xContent', required => false, trace => true},
);
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if
# the repo has a user passphrase defined - else pass undef
my $oFileIo = ref($xFile) ? $xFile : $self->openWrite($xFile);
# Determine size of content
my $lSize = defined($xContent) ? length(ref($xContent) ? $$xContent : $xContent) : 0;
# Write only if there is something to write
if ($lSize > 0)
{
$oFileIo->write(ref($xContent) ? $xContent : \$xContent);
}
# Else open the file so a zero length file is created (since file is not opened until first write)
else
{
$oFileIo->open();
}
# Close the file
$oFileIo->close();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'lSize', value => $lSize, trace => true},
);
}
####################################################################################################################################
# remove - remove path/file
####################################################################################################################################
@@ -647,7 +832,6 @@ sub remove
####################################################################################################################################
sub pathBase {shift->{strPathBase}}
sub driver {shift->{oDriver}}
sub cipherType {undef}
sub cipherPassUser {undef}
sub type {shift->{oDriver}->type()}
1;
test/lib/pgBackRestTest/Common/StoragePosix.pm
+1
View File
@@ -978,5 +978,6 @@ sub remove
sub className {STORAGE_POSIX_DRIVER}
sub tempExtension {shift->{strTempExtension}}
sub tempExtensionSet {my $self = shift; $self->{strTempExtension} = shift}
sub type {STORAGE_POSIX}
1;
test/lib/pgBackRestTest/Common/VmTest.pm
+1 -1
View File
@@ -509,7 +509,7 @@ sub vmValid
if (!defined($oyVm->{$strVm}))
{
confess &log(ERROR, "no definition for vm '${strVm}'", ERROR_OPTION_INVALID_VALUE);
confess &log(ERROR, "no definition for vm '${strVm}'");
}
}
test/lib/pgBackRestTest/Env/ConfigEnvTest.pm
-191
View File
@@ -1,191 +0,0 @@
####################################################################################################################################
# ConfigCommonTest.pm - Common code for Config unit tests
####################################################################################################################################
package pgBackRestTest::Env::ConfigEnvTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Getopt::Long qw(GetOptions);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::LibC qw(:test);
use pgBackRest::Version;
use pgBackRestTest::Common::RunTest;
use constant CONFIGENVTEST => 'ConfigEnvTest';
####################################################################################################################################
# Is the option secure?
####################################################################################################################################
sub optionTestSecure
{
my $self = shift;
my $strOption = shift;
return (cfgDefOptionSecure(cfgOptionId($strOption)) ? true : false);
}
sub optionTestSet
{
my $self = shift;
my $iOptionId = shift;
my $strValue = shift;
$self->{&CONFIGENVTEST}{option}{cfgOptionName($iOptionId)} = $strValue;
}
sub optionTestSetBool
{
my $self = shift;
my $iOptionId = shift;
my $bValue = shift;
$self->{&CONFIGENVTEST}{boolean}{cfgOptionName($iOptionId)} = defined($bValue) ? $bValue : true;
}
sub optionTestClear
{
my $self = shift;
my $iOptionId = shift;
delete($self->{&CONFIGENVTEST}{option}{cfgOptionName($iOptionId)});
delete($self->{&CONFIGENVTEST}{boolean}{cfgOptionName($iOptionId)});
}
sub configTestClear
{
my $self = shift;
my $rhConfig = $self->{&CONFIGENVTEST};
delete($self->{&CONFIGENVTEST});
return $rhConfig;
}
sub configTestSet
{
my $self = shift;
my $rhConfig = shift;
$self->{&CONFIGENVTEST} = $rhConfig;
}
####################################################################################################################################
# Write all secure options to a config file
####################################################################################################################################
sub configFileWrite
{
my $self = shift;
my $strConfigFile = shift;
my $rhConfig = shift;
my $strConfig = "[global]\n";
if (defined($rhConfig->{boolean}))
{
foreach my $strOption (sort(keys(%{$rhConfig->{boolean}})))
{
if ($self->optionTestSecure($strOption))
{
$strConfig .= "${strOption}=" . ($rhConfig->{boolean}{$strOption} ? 'y' : 'n') . "\n";
}
}
}
if (defined($rhConfig->{option}))
{
foreach my $strOption (sort(keys(%{$rhConfig->{option}})))
{
if ($self->optionTestSecure($strOption))
{
$strConfig .= "${strOption}=$rhConfig->{option}{$strOption}\n";
}
}
}
storageTest()->put($strConfigFile, $strConfig);
}
####################################################################################################################################
# Write all non-secure options to the command line
####################################################################################################################################
sub commandTestWrite
{
my $self = shift;
my $strCommand = shift;
my $strConfigFile = shift;
my $rhConfig = shift;
my @szyParam = ();
# Add boolean options
if (defined($rhConfig->{boolean}))
{
foreach my $strOption (sort(keys(%{$rhConfig->{boolean}})))
{
if (!$self->optionTestSecure($strOption))
{
if ($rhConfig->{boolean}{$strOption})
{
push(@szyParam, "--${strOption}");
}
else
{
push(@szyParam, "--no-${strOption}");
}
}
}
}
# Add non-boolean options
if (defined($rhConfig->{option}))
{
foreach my $strOption (sort(keys(%{$rhConfig->{option}})))
{
if (!$self->optionTestSecure($strOption))
{
push(@szyParam, "--${strOption}=$rhConfig->{option}{$strOption}");
}
}
}
# Add config file
push(@szyParam, '--' . cfgOptionName(CFGOPT_CONFIG) . "=${strConfigFile}");
# Add command
push(@szyParam, $strCommand);
return @szyParam;
}
####################################################################################################################################
# Load the configuration
####################################################################################################################################
sub configTestLoad
{
my $self = shift;
my $iCommandId = shift;
# A config file is required to store secure options before they can be parsed
my $strConfigFile = $self->testPath() . '/pgbackrest.test.conf';
$self->configFileWrite($strConfigFile, $self->{&CONFIGENVTEST});
my @stryArg = $self->commandTestWrite(cfgCommandName($iCommandId), $strConfigFile, $self->{&CONFIGENVTEST});
my $strConfigJson = cfgParseTest(projectBin(), join('|', @stryArg));
$self->testResult(
sub {configLoad(false, projectBin(), cfgCommandName($iCommandId), \$strConfigJson)},
true, 'config load: ' . join(" ", @stryArg));
}
1;
test/lib/pgBackRestTest/Env/ExpireEnvTest.pm
+27 -37
View File
@@ -20,10 +20,8 @@ use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
@@ -50,6 +48,7 @@ sub new
$self->{oHostBackup},
$self->{strBackRestExe},
$self->{oStorageRepo},
$self->{strPgPath},
$self->{oLogTest},
$self->{oRunTest},
) =
@@ -59,6 +58,7 @@ sub new
{name => 'oHostBackup', required => false, trace => true},
{name => 'strBackRestExe', trace => true},
{name => 'oStorageRepo', trace => true},
{name => 'strPgPath', trace => true},
{name => 'oLogTest', required => false, trace => true},
{name => 'oRunTest', required => false, trace => true},
);
@@ -124,7 +124,7 @@ sub info
logDebugParam
(
__PACKAGE__ . '->info', \@_,
{name => 'strDbPath', default => cfgOption(CFGOPT_PG_PATH)}
{name => 'strDbPath', default => $self->{strPgPath}}
);
# Open the control file and read system id and versions
@@ -168,8 +168,7 @@ sub info
ERROR,
'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} .
' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" .
'HINT: is this version of PostgreSQL supported?',
ERROR_VERSION_NOT_SUPPORTED);
'HINT: is this version of PostgreSQL supported?');
}
# Return from function and log return values if any
@@ -210,7 +209,6 @@ sub stanzaSet
my $oStanza = {};
my $oArchiveInfo = {};
my $oBackupInfo = {};
my $bEncrypted = defined($self->{oStorageRepo}->cipherType());
my $iArchiveDbId = 1;
my $iBackupDbId = 1;
@@ -218,26 +216,21 @@ sub stanzaSet
if (!$bStanzaUpgrade)
{
$oArchiveInfo =
new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE), false,
{bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_ARCHIVE : undef});
new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath(), false,
{bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef});
$oBackupInfo =
new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP), false,
{bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_MANIFEST : undef});
new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath(), false,
{bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef});
}
# Else get the info data from disk
else
{
$oArchiveInfo =
new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE),
{strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_ARCHIVE : undef});
new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath(),
{strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef});
$oBackupInfo =
new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP),
{strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_MANIFEST : undef});
}
if (cfgOption(CFGOPT_ONLINE))
{
confess &log(ERROR, "this function may not be used for online tests");
new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath(),
{strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef});
}
# Get the database info for the stanza
@@ -258,8 +251,8 @@ sub stanzaSet
$oBackupInfo->save();
# Get the archive and directory paths for the stanza
$$oStanza{strArchiveClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE) . '/' . ($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP);
$$oStanza{strArchiveClusterPath} = $self->{oHostBackup}->repoArchivePath($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oHostBackup}->repoBackupPath();
$self->{oStanzaHash}{$strStanza} = $oStanza;
@@ -291,14 +284,12 @@ sub stanzaCreate
my $strDbVersionTemp = $strDbVersion;
$strDbVersionTemp =~ s/\.//;
my $strDbPath = cfgOption(CFGOPT_PG_PATH);
# Create the test path for pg_control
storageTest()->pathCreate(($strDbPath . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true});
storageTest()->pathCreate(($self->{strPgPath} . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true});
# Generate pg_control for stanza-create
$self->controlGenerate($strDbPath, $strDbVersion);
executeTest('chmod 600 ' . $strDbPath . '/' . DB_FILE_PGCONTROL);
$self->controlGenerate($self->{strPgPath}, $strDbVersion);
executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
# Create the stanza and set the local stanza object
$self->stanzaSet($strStanza, $strDbVersion, false);
@@ -332,11 +323,11 @@ sub stanzaUpgrade
$strDbVersionTemp =~ s/\.//;
# Remove pg_control
storageTest()->remove(cfgOption(CFGOPT_PG_PATH) . '/' . DB_FILE_PGCONTROL);
storageTest()->remove($self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
# Copy pg_control for stanza-upgrade
$self->controlGenerate(cfgOption(CFGOPT_PG_PATH), $strDbVersion);
executeTest('chmod 600 ' . cfgOption(CFGOPT_PG_PATH) . '/' . DB_FILE_PGCONTROL);
$self->controlGenerate($self->{strPgPath}, $strDbVersion);
executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
$self->stanzaSet($strStanza, $strDbVersion, true);
@@ -393,7 +384,7 @@ sub backupCreate
# Get passphrase (returns undefined if repo not encrypted) to access the manifest
my $strCipherPassManifest =
(new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP)))->cipherPassSub();
(new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath()))->cipherPassSub();
my $strCipherPassBackupSet;
# If repo is encrypted then get passphrase for accessing the backup files from the last manifest if it exists provide one
@@ -540,7 +531,7 @@ sub archiveCreate
# Get passphrase (returns undefined if repo not encrypted) to access the archive files
my $strCipherPass =
(new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE)))->cipherPassSub();
(new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath()))->cipherPassSub();
push(my @stryArchive, $strArchive);
@@ -604,7 +595,7 @@ sub supplementalLog
join("\n", grep(!/^backup\.info.*$/i, storageRepo()->list("backup/${strStanza}"))));
# Output archive manifest
my $rhManifest = storageRepo()->manifest(STORAGE_REPO_ARCHIVE);
my $rhManifest = storageRepo()->manifest($self->{oHostBackup}->repoArchivePath());
my $strManifest;
my $strPrefix = '';
@@ -615,7 +606,7 @@ sub supplementalLog
if ($rhManifest->{$strEntry}->{type} eq 'd')
{
$strEntry = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE) . ($strEntry eq '.' ? '' : "/${strEntry}");
$strEntry = $self->{oHostBackup}->repoArchivePath($strEntry eq '.' ? undef : $strEntry);
# &log(WARN, "DIR $strEntry");
$strManifest .= (defined($strManifest) ? "\n" : '') . "${strEntry}:\n";
@@ -670,10 +661,9 @@ sub process
undef($$oStanza{strBackupDescription});
my $strCommand = $self->{strBackRestExe} .
' --' . cfgOptionName(CFGOPT_CONFIG) . '="' . $self->{oHostBackup}->backrestConfig() . '"' .
' --' . cfgOptionName(CFGOPT_STANZA) . '=' . $strStanza .
' --' . cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) . '=' . lc(DETAIL);
my $strCommand =
$self->{strBackRestExe} . ' --config="' . $self->{oHostBackup}->backrestConfig() . '"' . ' --stanza=' . $strStanza .
' --log-level-console=' . lc(DETAIL);
if (defined($iExpireFull))
{
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
+160 -128
View File
@@ -24,12 +24,11 @@ use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Host::HostS3Test;
@@ -50,6 +49,49 @@ use constant HOST_PATH_REPO => 'repo';
use constant HOST_PROTOCOL_TIMEOUT => 10;
push @EXPORT, qw(HOST_PROTOCOL_TIMEOUT);
####################################################################################################################################
# Configuration constants
####################################################################################################################################
use constant CFGDEF_SECTION_GLOBAL => 'global';
push @EXPORT, qw(CFGDEF_SECTION_GLOBAL);
use constant CFGDEF_SECTION_STANZA => 'stanza';
push @EXPORT, qw(CFGDEF_SECTION_STANZA);
use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL);
use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF);
use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR);
use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc';
push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
use constant STORAGE_CIFS => 'cifs';
push @EXPORT, qw(STORAGE_CIFS);
use constant STORAGE_S3 => 's3';
push @EXPORT, qw(STORAGE_S3);
use constant CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_DEFAULT);
use constant CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_IMMEDIATE);
use constant CFGOPTVAL_RESTORE_TYPE_NAME => 'name';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_NAME);
use constant CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_PRESERVE);
use constant CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_STANDBY);
use constant CFGOPTVAL_RESTORE_TYPE_TIME => 'time';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_TIME);
use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_XID);
use constant NONE => 'none';
push @EXPORT, qw(NONE);
use constant GZ => 'gz';
push @EXPORT, qw(GZ);
####################################################################################################################################
# new
####################################################################################################################################
@@ -251,8 +293,8 @@ sub backupEnd
{
if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink())
{
my $hTablespaceManifest = storageRepo()->manifest(
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
my $hTablespaceManifest = storageTest()->manifest(
$self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC));
# Remove . and ..
delete($hTablespaceManifest->{'.'});
@@ -305,19 +347,17 @@ sub backupEnd
}
# Else there should not be a tablespace directory at all. This is only valid for storage that supports links.
elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) &&
storageTest()->pathExists(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)))
storageTest()->pathExists($self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)))
{
confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
}
}
# Check that latest link exists unless repo links are disabled
my $strLatestLink = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST);
my $strLatestLink = $self->repoBackupPath(LINK_LATEST);
my $bLatestLinkExists = storageRepo()->exists($strLatestLink);
if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq CFGOPTVAL_REPO_TYPE_POSIX) && $self->hasLink())
if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq STORAGE_POSIX) && $self->hasLink())
{
my $strLatestLinkDestination = readlink($strLatestLink);
@@ -367,14 +407,12 @@ sub backupEnd
if ($self->synthetic() && $bManifestCompare)
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), undef,
$self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), undef,
${storageRepo()->get(
storageRepo()->openRead(
STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST,
{strCipherPass => $self->cipherPassManifest()}))});
$self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}))});
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
$self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo->get($self->repoBackupPath(FILE_BACKUP_INFO))});
}
}
@@ -446,8 +484,7 @@ sub backupCompare
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup;
my $oActualManifest = new pgBackRest::Manifest(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST),
{strCipherPass => $self->cipherPassManifest()});
$self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()});
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} =
$oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START);
@@ -478,7 +515,7 @@ sub backupCompare
# Determine repo size if compression or encryption is enabled
my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE};
if ($strCompressType ne CFGOPTVAL_COMPRESS_TYPE_NONE ||
if ($strCompressType ne NONE ||
(defined($oExpectedManifest->{&INI_SECTION_CIPHER}) &&
defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS})))
{
@@ -486,9 +523,9 @@ sub backupCompare
my $lRepoSize =
$oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
$oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
(storageRepo()->info(STORAGE_REPO_BACKUP .
"/${strBackup}/${strFileKey}" .
($strCompressType eq CFGOPTVAL_COMPRESS_TYPE_NONE ? '' : ".${strCompressType}")))->{size};
(storageRepo()->info(
$self->repoBackupPath("${strBackup}/${strFileKey}") .
($strCompressType eq NONE ? '' : ".${strCompressType}")))->{size};
if (defined($lRepoSize) &&
$lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
@@ -613,7 +650,7 @@ sub backupLast
my $self = shift;
my @stryBackup = storageRepo()->list(
STORAGE_REPO_BACKUP, {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'});
$self->repoBackupPath(), {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'});
if (!defined($stryBackup[0]))
{
@@ -780,13 +817,11 @@ sub stanzaCreate
if (defined($self->{oLogTest}) && $self->synthetic())
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
$self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo()->get($self->repoBackupPath(FILE_BACKUP_INFO))});
}
# Get the passphrase for accessing the manifest file
$self->{strCipherPassManifest} =
(new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza())))->cipherPassSub();
$self->{strCipherPassManifest} = (new pgBackRest::Backup::Info($self->repoBackupPath()))->cipherPassSub();
}
if (storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE))
@@ -795,13 +830,12 @@ sub stanzaCreate
if (defined($self->{oLogTest}) && $self->synthetic())
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
$self->repoArchivePath(ARCHIVE_INFO_FILE), undef, ${storageRepo()->get($self->repoArchivePath(ARCHIVE_INFO_FILE))});
}
# Get the passphrase for accessing the archived files
$self->{strCipherPassArchive} =
(new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza())))->cipherPassSub();
(new pgBackRest::Archive::Info($self->repoArchivePath()))->cipherPassSub();
}
# Return from function and log return values if any
@@ -848,16 +882,14 @@ sub stanzaUpgrade
storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO))
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
$self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo()->get($self->repoBackupPath(FILE_BACKUP_INFO))});
}
if (defined($self->{oLogTest}) && $self->synthetic() &&
storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE))
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
$self->repoArchivePath(ARCHIVE_INFO_FILE), undef, ${storageRepo()->get($self->repoArchivePath(ARCHIVE_INFO_FILE))});
}
# Return from function and log return values if any
@@ -1009,65 +1041,64 @@ sub configCreate
# General options
# ------------------------------------------------------------------------------------------------------------------------------
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE)} = lc(DETAIL);
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_FILE)} = testRunGet()->logLevelTestFile();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_STDERR)} = lc(OFF);
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_SUBPROCESS)} =
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-console'} = lc(DETAIL);
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-file'} = testRunGet()->logLevelTestFile();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-stderr'} = lc(OFF);
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-subprocess'} =
testRunGet()->logLevelTestFile() eq lc(OFF) ? 'n' : 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_TIMESTAMP)} = 'n';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_BUFFER_SIZE)} = '64k';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-timestamp'} = 'n';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'buffer-size'} = '64k';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_PATH)} = $self->logPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOCK_PATH)} = $self->lockPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_PROTOCOL_TIMEOUT)} = 60;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_DB_TIMEOUT)} = 45;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'protocol-timeout'} = 60;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'db-timeout'} = 45;
# Set to make sure that changing the default works and to speed compression for testing
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_LEVEL)} = 3;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level'} = 3;
# Only set network compress level if there is more than one host
if ($oHostBackup != $oHostDbMaster)
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_LEVEL_NETWORK)} = 1;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level-network'} = 1;
}
if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne CFGOPTVAL_COMPRESS_TYPE_GZ)
if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne 'gz')
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_TYPE)} = $oParam->{strCompressType};
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-type'} = $oParam->{strCompressType};
}
if ($self->isHostBackup())
{
if ($self->repoEncrypt())
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_CIPHER_TYPE)} =
CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_CIPHER_PASS)} = 'x';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-type'} = CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-pass'} = 'x';
}
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_PATH)} = $self->repoPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath();
# S3 settings
if ($oParam->{bS3})
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_TYPE)} = CFGOPTVAL_REPO_TYPE_S3;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_KEY)} = HOST_S3_ACCESS_KEY;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_KEY_SECRET)} = HOST_S3_ACCESS_SECRET_KEY;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_BUCKET)} = HOST_S3_BUCKET;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_ENDPOINT)} = HOST_S3_ENDPOINT;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_REGION)} = HOST_S3_REGION;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = STORAGE_S3;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key'} = HOST_S3_ACCESS_KEY;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key-secret'} = HOST_S3_ACCESS_SECRET_KEY;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-bucket'} = HOST_S3_BUCKET;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-endpoint'} = HOST_S3_ENDPOINT;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-region'} = HOST_S3_REGION;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'} = 'n';
}
if (defined($$oParam{bHardlink}) && $$oParam{bHardlink})
{
$self->{bHardLink} = true;
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_REPO_HARDLINK)} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'repo1-s3-hardlink'} = 'y';
}
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_ARCHIVE_COPY)} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_START_FAST)} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'archive-copy'} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y';
}
# Host specific options
@@ -1087,53 +1118,44 @@ sub configCreate
if ($self->nameTest(HOST_BACKUP))
{
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST)} = $oHostDb1->nameGet();
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_USER)} = $oHostDb1->userGet();
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_CMD)} = $oHostDb1->backrestExe();
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_CONFIG)} = $oHostDb1->backrestConfig();
$oParamHash{$strStanza}{'pg1-host'} = $oHostDb1->nameGet();
$oParamHash{$strStanza}{'pg1-host-user'} = $oHostDb1->userGet();
$oParamHash{$strStanza}{'pg1-host-cmd'} = $oHostDb1->backrestExe();
$oParamHash{$strStanza}{'pg1-host-config'} = $oHostDb1->backrestConfig();
# Port can't be configured for a synthetic host
if (!$self->synthetic())
{
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PORT)} = $oHostDb1->pgPort();
$oParamHash{$strStanza}{'pg1-port'} = $oHostDb1->pgPort();
}
}
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $oHostDb1->dbBasePath();
$oParamHash{$strStanza}{'pg1-path'} = $oHostDb1->dbBasePath();
if (defined($oHostDb2))
{
# Add an invalid replica to simulate more than one replica. A warning should be thrown when a stanza is created and a
# valid replica should be chosen.
my $iInvalidReplica = 2;
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iInvalidReplica))} = BOGUS;
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iInvalidReplica))} =
$oHostDb2->userGet();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iInvalidReplica))} =
$oHostDb2->backrestExe();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iInvalidReplica))} =
$oHostDb2->backrestConfig();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iInvalidReplica))} =
$oHostDb2->dbBasePath();
$oParamHash{$strStanza}{"pg2-host"} = BOGUS;
$oParamHash{$strStanza}{"pg2-host-user"} = $oHostDb2->userGet();
$oParamHash{$strStanza}{"pg2-host-cmd"} = $oHostDb2->backrestExe();
$oParamHash{$strStanza}{"pg2-host-config"} = $oHostDb2->backrestConfig();
$oParamHash{$strStanza}{"pg2-path"} = $oHostDb2->dbBasePath();
# Set a flag so we know there's a bogus host
$self->{bBogusHost} = true;
# Set a valid replica to the last possible index to ensure skipping indexes does not make a difference.
my $iValidReplica = cfgOptionIndexTotal(CFGOPT_PG_PATH);
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iValidReplica))} = $oHostDb2->nameGet();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iValidReplica))} =
$oHostDb2->userGet();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iValidReplica))} =
$oHostDb2->backrestExe();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iValidReplica))} =
$oHostDb2->backrestConfig();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iValidReplica))} = $oHostDb2->dbBasePath();
# Set a valid replica to a higher index to ensure skipping indexes does not make a difference
$oParamHash{$strStanza}{"pg8-host"} = $oHostDb2->nameGet();
$oParamHash{$strStanza}{"pg8-host-user"} = $oHostDb2->userGet();
$oParamHash{$strStanza}{"pg8-host-cmd"} = $oHostDb2->backrestExe();
$oParamHash{$strStanza}{"pg8-host-config"} = $oHostDb2->backrestConfig();
$oParamHash{$strStanza}{"pg8-path"} = $oHostDb2->dbBasePath();
# Only test explicit ports on the backup server. This is so locally configured ports are also tested.
if (!$self->synthetic() && $self->nameTest(HOST_BACKUP))
{
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iValidReplica))} = $oHostDb2->pgPort();
$oParamHash{$strStanza}{"pg8-port"} = $oHostDb2->pgPort();
}
}
}
@@ -1141,32 +1163,31 @@ sub configCreate
# If this is a database host
if ($self->isHostDb())
{
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $self->dbBasePath();
$oParamHash{$strStanza}{'pg1-path'} = $self->dbBasePath();
if (!$self->synthetic())
{
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_SOCKET_PATH)} = $self->pgSocketPath();
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PORT)} = $self->pgPort();
$oParamHash{$strStanza}{'pg1-socket-path'} = $self->pgSocketPath();
$oParamHash{$strStanza}{'pg1-port'} = $self->pgPort();
}
if ($bArchiveAsync)
{
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':' .
cfgCommandName(CFGCMD_ARCHIVE_PUSH)}{cfgOptionName(CFGOPT_ARCHIVE_ASYNC)} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':archive-push'}{'archive-async'} = 'y';
}
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_SPOOL_PATH)} = $self->spoolPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'spool-path'} = $self->spoolPath();
# If the the backup host is remote
if (!$self->isHostBackup())
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST)} = $oHostBackup->nameGet();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_USER)} = $oHostBackup->userGet();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_CMD)} = $oHostBackup->backrestExe();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_CONFIG)} = $oHostBackup->backrestConfig();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host'} = $oHostBackup->nameGet();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-user'} = $oHostBackup->userGet();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cmd'} = $oHostBackup->backrestExe();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-config'} = $oHostBackup->backrestConfig();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_PATH)} = $self->logPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOCK_PATH)} = $self->lockPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath();
}
}
@@ -1244,7 +1265,7 @@ sub manifestMunge
{name => 'bCache', default => true},
);
$self->infoMunge(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true);
$self->infoMunge($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true);
# Return from function and log return values if any
return logDebugReturn($strOperation);
@@ -1271,7 +1292,7 @@ sub manifestRestore
{name => 'bSave', default => true},
);
$self->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $bSave);
$self->infoRestore($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $bSave);
# Return from function and log return values if any
return logDebugReturn($strOperation);
@@ -1309,15 +1330,16 @@ sub infoMunge
# If the original file content does not exist then load it
if (!defined($self->{hInfoFile}{$strFileName}))
{
$self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini($strFileName, {oStorage => storageRepo(),
strCipherPass => !$bManifest ? storageRepo()->cipherPassUser() : $self->cipherPassManifest()});
$self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini(
storageRepo(), $strFileName,
{strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()});
}
# Make a copy of the original file contents
my $oMungeIni = new pgBackRest::Common::Ini(
$strFileName,
{bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), oStorage => storageRepo(),
strCipherPass => !$bManifest ? storageRepo()->cipherPassUser() : $self->cipherPassManifest()});
storageRepo(), $strFileName,
{bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}),
strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()});
# Load params
foreach my $strSection (keys(%{$hParam}))
@@ -1438,7 +1460,7 @@ sub configRecovery
if (@stryRecoveryOption)
{
$oConfig->{$strStanza}{cfgOptionName(CFGOPT_RECOVERY_OPTION)} = \@stryRecoveryOption;
$oConfig->{$strStanza}{'recovery-option'} = \@stryRecoveryOption;
}
# Save db config file
@@ -1472,7 +1494,7 @@ sub configRemap
}
# Rewrite recovery section
delete($oConfig->{"${strStanza}:restore"}{cfgOptionName(CFGOPT_TABLESPACE_MAP)});
delete($oConfig->{"${strStanza}:restore"}{'tablespace-map'});
my @stryTablespaceMap;
foreach my $strRemap (sort(keys(%$oRemapHashRef)))
@@ -1481,13 +1503,13 @@ sub configRemap
if ($strRemap eq MANIFEST_TARGET_PGDATA)
{
$oConfig->{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $strRemapPath;
$oConfig->{$strStanza}{'pg1-path'} = $strRemapPath;
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} = $strRemapPath;
if (defined($oHostBackup))
{
$oRemoteConfig->{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $strRemapPath;
$oRemoteConfig->{$strStanza}{'pg1-path'} = $strRemapPath;
}
}
else
@@ -1502,7 +1524,7 @@ sub configRemap
if (@stryTablespaceMap)
{
$oConfig->{"${strStanza}:restore"}{cfgOptionName(CFGOPT_TABLESPACE_MAP)} = \@stryTablespaceMap;
$oConfig->{"${strStanza}:restore"}{'tablespace-map'} = \@stryTablespaceMap;
}
# Save db config file
@@ -1577,14 +1599,13 @@ sub restore
$strComment = 'restore' .
($bDelta ? ' delta' : '') .
($bForce ? ', force' : '') .
($strBackup ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? ", backup '${strBackup}'" : '') .
($strBackup ne 'latest' ? ", backup '${strBackup}'" : '') .
# This does not output 'default' for synthetic tests to make expect logs match up (may change later)
($strType ? ", type '${strType}'" : (defined($rhExpectedManifest) ? '' : ", type 'default'")) .
($strTarget ? ", target '${strTarget}'" : '') .
($strTargetTimeline ? ", timeline '${strTargetTimeline}'" : '') .
($bTargetExclusive ? ', exclusive' : '') .
(defined($strTargetAction) && $strTargetAction ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_TARGET_ACTION)
? ', ' . cfgOptionName(CFGOPT_TARGET_ACTION) . "=${strTargetAction}" : '') .
(defined($strTargetAction) && $strTargetAction ne 'pause' ? ", target-action=${strTargetAction}" : '') .
(defined($rhRemapHash) ? ', remap' : '') .
(defined($iExpectedExitStatus) ? ", expect exit ${iExpectedExitStatus}" : '') .
(defined($strComment) ? " - ${strComment}" : '') .
@@ -1602,7 +1623,7 @@ sub restore
# - which should be the backup passed as strBackupExpected. If it is not defined, then set it based on the strBackup passed.
if (!defined($strBackupExpected))
{
$strBackupExpected = $strBackup eq cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? $oHostBackup->backupLast() :
$strBackupExpected = $strBackup eq 'latest' ? $oHostBackup->backupLast() :
$strBackup;
}
@@ -1610,9 +1631,7 @@ sub restore
{
# Load the manifest from the backup expected to be chosen/processed by restore
my $oExpectedManifest = new pgBackRest::Manifest(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} . $strBackupExpected. qw{/} .
FILE_MANIFEST),
$self->repoBackupPath($strBackupExpected . qw{/} . FILE_MANIFEST),
{strCipherPass => $oHostBackup->cipherPassManifest()});
$rhExpectedManifest = $oExpectedManifest->{oContent};
@@ -1672,7 +1691,7 @@ sub restore
' --config=' . $self->backrestConfig() .
($bDelta ? ' --delta' : '') .
($bForce ? ' --force' : '') .
($strBackup ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? " --set=${strBackup}" : '') .
($strBackup ne 'latest' ? " --set=${strBackup}" : '') .
(defined($strOptionalParam) ? " ${strOptionalParam} " : '') .
(defined($strType) && $strType ne CFGOPTVAL_RESTORE_TYPE_DEFAULT ? " --type=${strType}" : '') .
(defined($strTarget) ? " --target=\"${strTarget}\"" : '') .
@@ -1680,8 +1699,7 @@ sub restore
($bTargetExclusive ? ' --target-exclusive' : '') .
(defined($strLinkMap) ? $strLinkMap : '') .
($self->synthetic() ? '' : ' --link-all') .
(defined($strTargetAction) && $strTargetAction ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_TARGET_ACTION)
? ' --' . cfgOptionName(CFGOPT_TARGET_ACTION) . "=${strTargetAction}" : '') .
(defined($strTargetAction) && $strTargetAction ne 'pause' ? " --target-action=${strTargetAction}" : '') .
' --stanza=' . $self->stanza() . ' restore',
{strComment => $strComment, iExpectedExitStatus => $iExpectedExitStatus, oLogTest => $self->{oLogTest},
bLogOutput => $self->synthetic()},
@@ -1723,9 +1741,8 @@ sub restoreCompare
{
my $oExpectedManifest =
new pgBackRest::Manifest(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} . ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) .
'/'. FILE_MANIFEST),
$self->repoBackupPath(
($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST),
{strCipherPass => $oHostBackup->cipherPassManifest()});
# Get the --delta option from the backup manifest so the actual manifest can be built the same way for comparison
@@ -1734,9 +1751,8 @@ sub restoreCompare
$oLastManifest =
new pgBackRest::Manifest(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} .
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST),
$self->repoBackupPath(
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST),
{strCipherPass => $oHostBackup->cipherPassManifest()});
}
@@ -2027,6 +2043,20 @@ sub restoreCompare
storageTest()->remove("${strTestPath}/actual.manifest");
}
####################################################################################################################################
# Get repo backup/archive path
####################################################################################################################################
sub repoSubPath
{
my $self = shift;
my $strSubPath = shift;
my $strPath = shift;
return
($self->{strRepoPath} eq '/' ? '' : $self->{strRepoPath}) . "/${strSubPath}/" . $self->stanza() .
(defined($strPath) ? "/${strPath}" : '');
}
####################################################################################################################################
# Getters
####################################################################################################################################
@@ -2036,13 +2066,15 @@ sub backrestExe {return testRunGet()->backrestExe()}
sub bogusHost {return shift->{bBogusHost}}
sub hardLink {return shift->{bHardLink}}
sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)}
sub isFS {storageRepo()->type() ne STORAGE_S3}
sub isFS {storageRepo()->type() ne STORAGE_OBJECT}
sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()}
sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER}
sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY}
sub isHostDb {my $self = shift; return $self->isHostDbMaster() || $self->isHostDbStandby()}
sub lockPath {return shift->{strLockPath}}
sub logPath {return shift->{strLogPath}}
sub repoArchivePath {return shift->repoSubPath('archive', shift)}
sub repoBackupPath {return shift->repoSubPath('backup', shift)}
sub repoPath {return shift->{strRepoPath}}
sub repoEncrypt {return shift->{bRepoEncrypt}}
sub stanza {return testRunGet()->stanza()}
test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm
+1 -1
View File
@@ -17,7 +17,7 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use pgBackRest::Common::Log;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm
+1 -2
View File
@@ -21,10 +21,9 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm
+1 -1
View File
@@ -23,7 +23,7 @@ use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
test/lib/pgBackRestTest/Env/Host/HostDbTest.pm
+1 -1
View File
@@ -22,7 +22,7 @@ use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
test/lib/pgBackRestTest/Env/Host/HostS3Test.pm
+1 -2
View File
@@ -22,9 +22,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBaseTest;
test/lib/pgBackRestTest/Env/HostEnvTest.pm
+11 -36
View File
@@ -2,7 +2,7 @@
# FullCommonTest.pm - Common code for backup tests
####################################################################################################################################
package pgBackRestTest::Env::HostEnvTest;
use parent 'pgBackRestTest::Env::ConfigEnvTest';
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
@@ -18,9 +18,9 @@ use Storable qw(dclone);
use pgBackRest::Archive::Common;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
@@ -43,11 +43,6 @@ use constant ENCRYPTION_KEY_MANIFEST => 'manifest';
use constant ENCRYPTION_KEY_BACKUPSET => 'backupset';
push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET);
use constant NONE => CFGOPTVAL_COMPRESS_TYPE_NONE;
push @EXPORT, qw(NONE);
use constant GZ => CFGOPTVAL_COMPRESS_TYPE_GZ;
push @EXPORT, qw(GZ);
####################################################################################################################################
# setup
####################################################################################################################################
@@ -151,6 +146,13 @@ sub setup
$oHostBackup = $strBackupDestination eq HOST_DB_MASTER ? $oHostDbMaster : $oHostDbStandby;
}
storageRepoCommandSet(
$self->backrestExeHelper() .
' --config=' . $oHostBackup->backrestConfig() . ' --stanza=' . $self->stanza() . ' --log-level-console=off' .
' --log-level-stderr=error' .
($oConfigParam->{bS3} ? ' --no-repo1-s3-verify-tls --repo1-s3-host=' . $oHostS3->ipGet() : ''),
$oConfigParam->{bS3} ? STORAGE_OBJECT : STORAGE_POSIX);
# Create db-standby config
if (defined($oHostDbStandby))
{
@@ -161,37 +163,10 @@ sub setup
bArchiveAsync => $$oConfigParam{bArchiveAsync}});
}
# Set options needed for storage helper
$self->optionTestSet(CFGOPT_PG_PATH, $oHostDbMaster->dbBasePath());
$self->optionTestSet(CFGOPT_REPO_PATH, $oHostBackup->repoPath());
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
# Configure the repo to be encrypted if required
if ($bRepoEncrypt)
{
$self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
$self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, 'x');
}
# Set S3 options
if (defined($oHostS3))
{
$self->optionTestSet(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3);
$self->optionTestSet(CFGOPT_REPO_S3_KEY, HOST_S3_ACCESS_KEY);
$self->optionTestSet(CFGOPT_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY);
$self->optionTestSet(CFGOPT_REPO_S3_BUCKET, HOST_S3_BUCKET);
$self->optionTestSet(CFGOPT_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT);
$self->optionTestSet(CFGOPT_REPO_S3_REGION, HOST_S3_REGION);
$self->optionTestSet(CFGOPT_REPO_S3_HOST, $oHostS3->ipGet());
$self->optionTestSetBool(CFGOPT_REPO_S3_VERIFY_TLS, false);
}
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Create S3 bucket
if (defined($oHostS3))
{
storageRepo()->{oStorageC}->bucketCreate();
storageRepo()->create();
}
return $oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3;
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
+39 -44
View File
@@ -21,10 +21,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
@@ -93,12 +91,12 @@ sub run
# without slowing down the other tests too much.
if ($bS3)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
# Reduce log level to warn because parallel tests do not create deterministic logs
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(WARN)}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(WARN)}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(WARN)}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(WARN)}});
}
# Get base time
@@ -115,7 +113,7 @@ sub run
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 16384;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::false;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = CFGOPTVAL_COMPRESS_TYPE_NONE;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = NONE;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL} = 3;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK} = $bRemote ? 1 : 3;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::false;
@@ -306,7 +304,7 @@ sub run
my $strTestPoint;
# Create the archive info file
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-online'});
# Create a link to postgresql.conf
storageTest()->pathCreate($oHostDbMaster->dbPath() . '/pg_config', {strMode => '0700', bCreateParent => true});
@@ -343,14 +341,13 @@ sub run
{oExpectedManifest => \%oManifest,
strOptionalParam => $strOptionalParam .
# Pass ssh path to make sure it is used
($bRemote ? ' --' . cfgOptionName(CFGOPT_CMD_SSH) . '=/usr/bin/ssh' : '') .
($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') .
# Pass bogus ssh port to make sure it is passed through the protocol layer (it won't be used)
($bRemote ? ' --' . cfgOptionName(CFGOPT_PG_PORT) . '=9999' : '') .
($bRemote ? ' --pg1-port=9999' : '') .
# Pass bogus socket path to make sure it is passed through the protocol layer (it won't be used)
($bRemote ? ' --' . cfgOptionName(CFGOPT_PG_SOCKET_PATH) . '=/test_socket_path' : '') .
' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=16384 --' . cfgOptionName(CFGOPT_CHECKSUM_PAGE) .
' --' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1',
strRepoType => $bS3 ? undef : CFGOPTVAL_REPO_TYPE_CIFS, strTest => $strTestPoint, fTestDelay => 0});
($bRemote ? ' --pg1-socket-path=/test_socket_path' : '') .
' --buffer-size=16384 --checksum-page --process-max=1',
strRepoType => $bS3 ? undef : STORAGE_CIFS, strTest => $strTestPoint, fTestDelay => 0});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 65536;
@@ -409,13 +406,13 @@ sub run
# Resume by copying the valid full backup over the last aborted full backup if it exists, or by creating a new path
my $strResumeBackup = (storageRepo()->list(
STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))[0];
$oHostBackup->repoBackupPath(), {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))[0];
my $strResumeLabel = $strResumeBackup ne $strFullBackup ?
$strResumeBackup : backupLabel(storageRepo(), $strType, undef, time());
my $strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . '/' . $strResumeLabel);
$strResumeBackup : backupLabel(storageRepo(), $oHostBackup->repoBackupPath(), $strType, undef, time());
my $strResumePath = $oHostBackup->repoBackupPath($strResumeLabel);
forceStorageRemove(storageRepo(), $strResumePath, {bRecurse => true});
forceStorageMove(storageRepo(), 'backup/' . $self->stanza() . "/${strFullBackup}", $strResumePath);
forceStorageMove(storageRepo(), $oHostBackup->repoBackupPath($strFullBackup), $strResumePath);
# Set ownership on base directory to bogus values
if (!$bRemote)
@@ -464,7 +461,7 @@ sub run
# Create files to be excluded with the --exclude option
$oHostBackup->configUpdate(
{(CFGDEF_SECTION_GLOBAL . ':backup') =>
{cfgOptionName(CFGOPT_EXCLUDE) => ['postgresql.auto.conf', 'pg_log/', 'pg_log2', 'apipe']}});
{'exclude' => ['postgresql.auto.conf', 'pg_log/', 'pg_log2', 'apipe']}});
$oHostDbMaster->dbLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.auto.conf',
'../pg_config/postgresql.conf', true);
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_log');
@@ -476,7 +473,7 @@ sub run
$strFullBackup = $oHostBackup->backup(
$strType, 'resume',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--force --' . cfgOptionName(CFGOPT_CHECKSUM_PAGE) . ($bDeltaBackup ? ' --delta' : '')});
strOptionalParam => '--force --checksum-page' . ($bDeltaBackup ? ' --delta' : '')});
# Remove postmaster.pid so restore will succeed (the rest will be cleaned up by the delta)
storageTest->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
@@ -523,7 +520,7 @@ sub run
strOptionalParam => ' --link-all' . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '')});
# Remove excludes now that they just create noise in the log
$oHostBackup->configUpdate({(CFGDEF_SECTION_GLOBAL . ':backup') => {cfgOptionName(CFGOPT_EXCLUDE) => []}});
$oHostBackup->configUpdate({(CFGDEF_SECTION_GLOBAL . ':backup') => {'exclude' => []}});
# Run again to fix permissions
if (!$bRemote)
@@ -624,11 +621,11 @@ sub run
$strType = CFGOPTVAL_BACKUP_TYPE_INCR;
# Create resumable backup from last backup
$strResumeLabel = backupLabel(storageRepo(), $strType, substr($strBackup, 0, 16), time());
$strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . '/' . $strResumeLabel);
$strResumeLabel = backupLabel(storageRepo(), $oHostBackup->repoBackupPath(), $strType, substr($strBackup, 0, 16), time());
$strResumePath = $oHostBackup->repoBackupPath($strResumeLabel);
forceStorageRemove(storageRepo(), $strResumePath);
forceStorageMove(storageRepo(), 'backup/' . $self->stanza() . "/${strBackup}", $strResumePath);
forceStorageMove(storageRepo(), $oHostBackup->repoBackupPath($strBackup), $strResumePath);
# Munge manifest so the resumed file in the repo appears to be bad
if ($bEncrypt || $bRemote)
@@ -698,7 +695,7 @@ sub run
$strBackup = $oHostBackup->backup(
$strType, 'resume and add tablespace 2',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')});
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@@ -719,9 +716,7 @@ sub run
$strBackup = $oHostBackup->backup(
$strType, 'drop tablespace 11',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' .
($bDeltaBackup ? ' --delta' : '')});
{oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@@ -776,7 +771,7 @@ sub run
$strBackup = $oHostBackup->backup(
$strType, 'add files and remove tablespace 2',
{oExpectedManifest => \%oManifest, strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1'});
{oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1'});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@@ -801,7 +796,7 @@ sub run
$strBackup = $oHostBackup->backup(
$strType, 'updates since last full', {oExpectedManifest => \%oManifest,
strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')});
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@@ -814,17 +809,17 @@ sub run
# Enable compression to ensure a warning is raised (reset when gz to avoid log churn since it is the default)
if ($strCompressType eq GZ)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => undef}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'compress-type' => undef}});
}
else
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => $strCompressType}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'compress-type' => $strCompressType}});
}
# Enable hardlinks (except for s3) to ensure a warning is raised
if (!$bS3)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_REPO_HARDLINK) => 'y'}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'repo1-hardlink' => 'y'}});
}
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1;
@@ -840,7 +835,7 @@ sub run
$oHostBackup->backup(
$strType, 'remove files',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')});
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@@ -889,8 +884,7 @@ sub run
$strFullBackup, {&MANIFEST_SECTION_BACKUP_OPTION => {&MANIFEST_KEY_CHECKSUM_PAGE => undef}}, false);
$strBackup = $oHostBackup->backup(
$strType, 'add file',
{oExpectedManifest => \%oManifest, strOptionalParam => '--' . cfgOptionName(CFGOPT_CHECKSUM_PAGE)});
$strType, 'add file', {oExpectedManifest => \%oManifest, strOptionalParam => '--checksum-page'});
# Selective Restore
#---------------------------------------------------------------------------------------------------------------------------
@@ -906,7 +900,7 @@ sub run
{&MANIFEST_SUBKEY_CHECKSUM});
$oHostDbMaster->restore(
'selective restore 16384', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'selective restore 16384', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true,
strOptionalParam => '--db-include=16384'});
@@ -924,7 +918,7 @@ sub run
delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/16384/17000'}{&MANIFEST_SUBKEY_CHECKSUM});
$oHostDbMaster->restore(
'selective restore 32768', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'selective restore 32768', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true,
strOptionalParam => '--db-include=32768'});
@@ -932,12 +926,12 @@ sub run
'7579ada0808d7f98087a0a586d0df9de009cdc33';
$oHostDbMaster->restore(
'error on invalid id', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'error on invalid id', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true,
iExpectedExitStatus => ERROR_DB_MISSING, strOptionalParam => '--log-level-console=warn --db-include=7777'});
$oHostDbMaster->restore(
'error on system id', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'error on system id', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true,
iExpectedExitStatus => ERROR_DB_INVALID, strOptionalParam => '--log-level-console=warn --db-include=1'});
@@ -952,7 +946,7 @@ sub run
delete($oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'});
$oHostDbMaster->restore(
'no tablespace remap', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'no tablespace remap', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bTablespace => false,
strOptionalParam => '--tablespace-map-all=../../tablespace'});
@@ -965,8 +959,9 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
if (!$bRemote && !$bS3)
{
executeTest('ls -1Rtr ' . storageRepo()->pathGet('backup/' . $self->stanza() . '/' . PATH_BACKUP_HISTORY),
{oLogTest => $self->expect(), bRemote => $bRemote});
executeTest(
'ls -1Rtr ' . $oHostBackup->repoBackupPath(PATH_BACKUP_HISTORY),
{oLogTest => $self->expect(), bRemote => $bRemote});
}
# Test backup from standby warning that standby not configured so option reset
@@ -975,7 +970,7 @@ sub run
{
$strBackup = $oHostBackup->backup(
$strType, 'option backup-standby reset - backup performed from master', {oExpectedManifest => \%oManifest,
strOptionalParam => '--log-level-console=info --' . cfgOptionName(CFGOPT_BACKUP_STANDBY)});
strOptionalParam => '--log-level-console=info --backup-standby'});
}
}
}
test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm
+6 -9
View File
@@ -20,12 +20,11 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::VmTest;
@@ -86,7 +85,7 @@ sub run
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
# Create the archive info file
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-online'});
# Push a WAL segment
&log(INFO, ' push first WAL');
@@ -96,7 +95,7 @@ sub run
if ($iError == 0)
{
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
$oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'},
&INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}});
}
@@ -120,13 +119,12 @@ sub run
# Fix the database version
if ($iError == 0)
{
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE));
}
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageRepo()->list(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))},
"000000010000000100000001-${strWalHash}${strCompressExt}",
'segment 2-4 not pushed', {iWaitSeconds => 5});
@@ -134,8 +132,7 @@ sub run
$oHostDbMaster->archivePush($strWalPath, $strWalTestFile, 5);
$self->testResult(
sub {storageRepo()->list(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))},
"(000000010000000100000001-${strWalHash}${strCompressExt}, " .
"000000010000000100000005-${strWalHash}${strCompressExt})",
'segment 5 is pushed', {iWaitSeconds => 5});
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
+20 -21
View File
@@ -20,11 +20,10 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
@@ -38,13 +37,15 @@ use pgBackRestTest::Common::VmTest;
sub archiveCheck
{
my $self = shift;
my $oHostBackup = shift;
my $strArchiveFile = shift;
my $strArchiveChecksum = shift;
my $strCompressType = shift;
my $strSpoolPath = shift;
# Build the archive name to check for at the destination
my $strArchiveCheck = PG_VERSION_94 . "-1/${strArchiveFile}-${strArchiveChecksum}";
my $strArchiveCheck = $oHostBackup->repoArchivePath(
PG_VERSION_94 . "-1/" . substr($strArchiveFile, 0, 16) . "/${strArchiveFile}-${strArchiveChecksum}");
if (defined($strCompressType))
{
@@ -56,13 +57,13 @@ sub archiveCheck
do
{
$bFound = storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}");
$bFound = storageRepo()->exists($strArchiveCheck);
}
while (!$bFound && waitMore($oWait));
if (!$bFound)
{
confess 'unable to find ' . storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}");
confess "unable to find ${strArchiveCheck}";
}
if (defined($strSpoolPath))
@@ -108,7 +109,7 @@ sub run
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE});
# Reduce console logging to detail
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(DETAIL)}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(DETAIL)}});
# Create the wal path
my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
@@ -121,11 +122,11 @@ sub run
# Create archive-push command
my $strCommandPush =
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=' . $self->stanza() .
' ' . cfgCommandName(CFGCMD_ARCHIVE_PUSH);
' archive-push';
my $strCommandGet =
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=' . $self->stanza() .
' ' . cfgCommandName(CFGCMD_ARCHIVE_GET);
' archive-get';
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' archive.info missing');
@@ -142,9 +143,7 @@ sub run
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
#---------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaCreate(
'stanza create',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('stanza create', {strOptionalParam => '--no-online'});
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' push first WAL');
@@ -160,7 +159,7 @@ sub run
push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}");
# Test that the WAL was pushed
$self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType);
$self->archiveCheck($oHostBackup, $strSourceFile, $strArchiveChecksum, $strCompressType);
# Remove from archive_status
storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready");
@@ -245,7 +244,7 @@ sub run
}
# Test that the WAL was pushed
$self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbMaster->spoolPath());
$self->archiveCheck($oHostBackup, $strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbMaster->spoolPath());
# Remove from archive_status
storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready");
@@ -263,7 +262,7 @@ sub run
"${strCommandPush} --archive-async ${strWalPath}/00000002.history",
{oLogTest => $self->expect()});
if (!storageRepo()->exists(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/00000002.history'))
if (!storageRepo()->exists($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/00000002.history')))
{
confess 'unable to find history file in archive';
}
@@ -275,7 +274,7 @@ sub run
# db section and corresponding history munged
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
$oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB_HISTORY => {'1' => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}});
$oHostDbMaster->executeSimple(
@@ -283,13 +282,13 @@ sub run
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
# Restore the file to its original condition
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE));
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' db system-id mismatch error');
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
$oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000},
&INFO_ARCHIVE_SECTION_DB_HISTORY => {'1' => {&INFO_ARCHIVE_KEY_DB_ID => 5000900090001855000}}});
@@ -302,7 +301,7 @@ sub run
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
# Restore the file to its original condition
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE));
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' stop');
@@ -384,7 +383,7 @@ sub run
$oHostDbMaster->executeSimple(
$strCommandPush . " ${strWalPath}/${strSourceFile}.partial",
{oLogTest => $self->expect()});
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum);
$self->archiveCheck($oHostBackup, "${strSourceFile}.partial", $strArchiveChecksum);
push(@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}");
@@ -393,7 +392,7 @@ sub run
$oHostDbMaster->executeSimple(
$strCommandPush . " ${strWalPath}/${strSourceFile}.partial", {oLogTest => $self->expect()});
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum);
$self->archiveCheck($oHostBackup, "${strSourceFile}.partial", $strArchiveChecksum);
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' .partial WAL with different checksum');
@@ -405,7 +404,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))},
'(' . join(', ', @stryExpectedWAL) . ')',
'all WAL in archive', {iWaitSeconds => 5});
}
test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm
+4 -43
View File
@@ -21,50 +21,17 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::VmTest;
use pgBackRestTest::Env::ExpireEnvTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostS3Test;
use pgBackRestTest::Env::HostEnvTest;
####################################################################################################################################
# initStanzaOption
####################################################################################################################################
sub initStanzaOption
{
my $self = shift;
my $strDbBasePath = shift;
my $strRepoPath = shift;
my $oHostS3 = shift;
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_PG_PATH, $strDbBasePath);
$self->optionTestSet(CFGOPT_REPO_PATH, $strRepoPath);
$self->optionTestSet(CFGOPT_LOG_PATH, $self->testPath());
$self->optionTestSetBool(CFGOPT_ONLINE, false);
$self->optionTestSet(CFGOPT_DB_TIMEOUT, 5);
$self->optionTestSet(CFGOPT_PROTOCOL_TIMEOUT, 6);
if (defined($oHostS3))
{
$self->optionTestSet(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3);
$self->optionTestSet(CFGOPT_REPO_S3_KEY, HOST_S3_ACCESS_KEY);
$self->optionTestSet(CFGOPT_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY);
$self->optionTestSet(CFGOPT_REPO_S3_BUCKET, HOST_S3_BUCKET);
$self->optionTestSet(CFGOPT_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT);
$self->optionTestSet(CFGOPT_REPO_S3_REGION, HOST_S3_REGION);
$self->optionTestSet(CFGOPT_REPO_S3_HOST, $oHostS3->ipGet());
$self->optionTestSetBool(CFGOPT_REPO_S3_VERIFY_TLS, false);
}
}
####################################################################################################################################
# run
####################################################################################################################################
@@ -103,12 +70,9 @@ sub run
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
true, $self->expect(), {bS3 => $bS3, bRepoEncrypt => $bEncrypt});
$self->initStanzaOption($oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3);
$self->configTestLoad(CFGCMD_STANZA_CREATE);
# Create the test object
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest(
$oHostBackup, $self->backrestExe(), storageRepo(), $self->expect(), $self);
$oHostBackup, $self->backrestExe(), storageRepo(), $oHostDbMaster->dbPath(), $self->expect(), $self);
$oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_92);
@@ -202,12 +166,9 @@ sub run
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
true, $self->expect(), {bS3 => $bS3, bRepoEncrypt => $bEncrypt});
$self->initStanzaOption($oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3);
$self->configTestLoad(CFGCMD_STANZA_CREATE);
# Create the test object
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest(
$oHostBackup, $self->backrestExe(), storageRepo(), $self->expect(), $self);
$oHostBackup, $self->backrestExe(), storageRepo(), $oHostDbMaster->dbPath(), $self->expect(), $self);
$oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_92);
test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm
+28 -34
View File
@@ -19,14 +19,13 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
@@ -40,17 +39,6 @@ sub run
{
my $self = shift;
# Archive and backup info file names
my $strArchiveInfoFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE;
my $strArchiveInfoCopyFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT;
my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old";
my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old";
my $strBackupInfoFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO;
my $strBackupInfoCopyFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT;
my $strBackupInfoOldFile = "${strBackupInfoFile}.old";
my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old";
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => true, compress => GZ},
@@ -80,9 +68,20 @@ sub run
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt,
strCompressType => $strCompressType});
# Archive and backup info file names
my $strArchiveInfoFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE);
my $strArchiveInfoCopyFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE . INI_COPY_EXT);
my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old";
my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old";
my $strBackupInfoFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO);
my $strBackupInfoCopyFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO . INI_COPY_EXT);
my $strBackupInfoOldFile = "${strBackupInfoFile}.old";
my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old";
# Create the stanza
$oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_MISSING,
strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_LOG_LEVEL_FILE) . '=info'});
strOptionalParam => '--no-online --log-level-file=info'});
# Generate pg_control for stanza-create
storageTest()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
@@ -91,17 +90,17 @@ sub run
# Fail stanza upgrade before stanza-create has been performed
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaUpgrade('fail on stanza not initialized since archive.info is missing',
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-online'});
# Create the stanza successfully without force
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-online'});
# Rerun stanza-create and confirm it does not fail
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaCreate(
'do not fail on rerun of stanza-create - info files exist and DB section ok',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
{strOptionalParam => '--no-online'});
# Stanza Create fails when not using force - database mismatch with pg_control file
#--------------------------------------------------------------------------------------------------------------------------
@@ -109,15 +108,14 @@ sub run
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
$oHostBackup->stanzaCreate('fail on database mismatch and warn force option deprecated',
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) .
' --' . cfgOptionName(CFGOPT_FORCE)});
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-online --force'});
# Restore pg_control
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_93);
# Perform a stanza upgrade which will indicate already up to date
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaUpgrade('already up to date', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaUpgrade('already up to date', {strOptionalParam => '--no-online'});
# Create the wal path
my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
@@ -140,7 +138,7 @@ sub run
if (!$bEncrypt)
{
$oHostBackup->stanzaCreate('fail on archive info file missing from non-empty dir',
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-online'});
}
# Restore info files from copy
@@ -170,8 +168,7 @@ sub run
# Save a pre-upgrade copy of archive info fo testing db-id mismatch
forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false});
$oHostBackup->stanzaUpgrade('successful upgrade creates additional history', {strOptionalParam => '--no-' .
cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaUpgrade('successful upgrade creates additional history', {strOptionalParam => '--no-online'});
# Make sure that WAL from the old version can still be retrieved
#--------------------------------------------------------------------------------------------------------------------------
@@ -193,7 +190,7 @@ sub run
# Push a WAL segment so have a valid file in the latest DB archive dir only
$oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1);
$self->testResult(
sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-2/0000000100000001')},
sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-2/0000000100000001'))},
'000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_94) . ".${strCompressType}",
'check that WAL is in the archive at -2');
@@ -201,8 +198,7 @@ sub run
#--------------------------------------------------------------------------------------------------------------------------
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_PGTBLSPC);
$oHostBackup->backup(
'full', 'create first full backup ',
{strOptionalParam => '--repo1-retention-full=2 --no-' . cfgOptionName(CFGOPT_ONLINE)}, false);
'full', 'create first full backup ', {strOptionalParam => '--repo1-retention-full=2 --no-online'}, false);
# Upgrade the stanza
#--------------------------------------------------------------------------------------------------------------------------
@@ -210,8 +206,7 @@ sub run
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_95);
forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
$oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-online'});
# Copy archive.info and restore really old version
forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false});
@@ -219,16 +214,15 @@ sub run
forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoFile, {bRecurse => false});
# Confirm versions
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza()));
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
my $oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath());
my $oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath());
$self->testResult(sub {$oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
PG_VERSION_93)}, true, 'archive at old pg version');
$self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef,
PG_VERSION_95)}, true, 'backup at new pg version');
$oHostBackup->stanzaUpgrade(
'upgrade fails with mismatched db-ids',
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
'upgrade fails with mismatched db-ids', {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-online'});
# Restore archive.info
forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false});
@@ -239,8 +233,8 @@ sub run
$oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1);
# Test backup is changed from type=DIFF to FULL (WARN message displayed)
my $oExecuteBackup = $oHostBackup->backupBegin('diff', 'diff changed to full backup',
{strOptionalParam => '--repo1-retention-full=2 --no-' . cfgOptionName(CFGOPT_ONLINE)});
my $oExecuteBackup = $oHostBackup->backupBegin(
'diff', 'diff changed to full backup', {strOptionalParam => '--repo1-retention-full=2 --no-online'});
$oHostBackup->backupEnd('full', $oExecuteBackup, undef, false);
# Delete the stanza
test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm
+2 -6
View File
@@ -16,7 +16,6 @@ use Storable qw(dclone);
use Time::HiRes qw(gettimeofday);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
@@ -63,11 +62,8 @@ sub run
for (my $iIndex = 0; $iIndex < $iRunTotal; $iIndex++)
{
executeTest(
$self->backrestExe() . ' --' . cfgOptionName(CFGOPT_STANZA) . '=' . $self->stanza() .
' --' . cfgOptionName(CFGOPT_ARCHIVE_ASYNC) .
' --' . cfgOptionName(CFGOPT_SPOOL_PATH) . '=' . $self->{strSpoolPath} .
' --' . cfgOptionName(CFGOPT_ARCHIVE_TIMEOUT) . '=1' .
' archive-push /pg_xlog/000000010000000100000001');
$self->backrestExe() . ' --stanza=' . $self->stanza() . ' --archive-async --spool-path=' . $self->{strSpoolPath} .
' --archive-timeout=1 archive-push /pg_xlog/000000010000000100000001');
}
&log(INFO, 'time per execution: ' . ((gettimeofday() - $lTimeBegin) / $iRunTotal));
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
+40 -58
View File
@@ -20,10 +20,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
@@ -111,8 +109,8 @@ sub run
# without slowing down the other tests too much.
if ($bS3)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
}
$oHostDbMaster->clusterCreate();
@@ -122,7 +120,7 @@ sub run
# Get passphrase to access the Manifest file from backup.info - returns undefined if repo not encrypted
my $strCipherPass =
(new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP)))->cipherPassSub();
(new pgBackRest::Backup::Info($oHostBackup->repoBackupPath()))->cipherPassSub();
# Create a manifest with the pg version to get version-specific paths
my $oManifest = new pgBackRest::Manifest(BOGUS, {bLoad => false, strDbVersion => $self->pgVersion(),
@@ -157,13 +155,13 @@ sub run
my $strComment = undef;
# Archive and backup info file names
my $strArchiveInfoFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE;
my $strArchiveInfoCopyFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT;
my $strArchiveInfoFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE);
my $strArchiveInfoCopyFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE . INI_COPY_EXT);
my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old";
my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old";
my $strBackupInfoFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO;
my $strBackupInfoCopyFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT;
my $strBackupInfoFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO);
my $strBackupInfoCopyFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO . INI_COPY_EXT);
my $strBackupInfoOldFile = "${strBackupInfoFile}.old";
my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old";
@@ -232,7 +230,7 @@ sub run
# load the archive info file and munge it for testing by breaking the database version
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
$oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'},
&INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}});
@@ -245,7 +243,7 @@ sub run
}
# Restore the file to its original condition
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE));
# Check archive_timeout error when WAL segment is not found
$strComment = 'fail on archive timeout';
@@ -268,7 +266,7 @@ sub run
# Load the backup.info file and munge it for testing by breaking the database version and system id
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
$oHostBackup->repoBackupPath(FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB =>
{&INFO_BACKUP_KEY_DB_VERSION => '8.0', &INFO_BACKUP_KEY_SYSTEM_ID => 6999999999999999999},
&INFO_BACKUP_SECTION_DB_HISTORY =>
@@ -284,7 +282,7 @@ sub run
}
# Restore the file to its original condition
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO));
$oHostBackup->infoRestore($oHostBackup->repoBackupPath(FILE_BACKUP_INFO));
# ??? Removed temporarily until manifest build can be brought back into the check command
# Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check
@@ -332,22 +330,20 @@ sub run
# Run stanza-create online to confirm proper handling of configValidation error against new pg-path
$oHostBackup->stanzaCreate('fail on database mismatch with directory',
{strOptionalParam => ' --' . cfgOptionName(CFGOPT_PG_PATH) . '=' . $oHostDbMaster->dbPath() .
'/testbase/', iExpectedExitStatus => ERROR_DB_MISMATCH});
{strOptionalParam => ' --pg1-path=' . $oHostDbMaster->dbPath() . '/testbase/',
iExpectedExitStatus => ERROR_DB_MISMATCH});
# Remove the directories to be able to create the stanza
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP, {bRecurse => true});
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE, {bRecurse => true});
forceStorageRemove(storageRepo(), $oHostBackup->repoBackupPath(), {bRecurse => true});
forceStorageRemove(storageRepo(), $oHostBackup->repoArchivePath(), {bRecurse => true});
# Stanza Upgrade - tests configValidate code - all other tests in synthetic integration tests
#-----------------------------------------------------------------------------------------------------------------------
# Run stanza-create offline to create files needing to be upgraded (using new pg-path)
$oHostBackup->stanzaCreate('successfully create stanza files to be upgraded',
{strOptionalParam =>
' --' . cfgOptionName(CFGOPT_PG_PATH) . '=' . $oHostDbMaster->dbPath() .
'/testbase/ --no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza()));
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
{strOptionalParam => ' --pg1-path=' . $oHostDbMaster->dbPath() . '/testbase/ --no-online --force'});
my $oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath());
my $oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath());
# Read info files to confirm the files were created with a different database version
if ($self->pgVersion() eq PG_VERSION_94)
@@ -369,8 +365,8 @@ sub run
$oHostBackup->stanzaUpgrade('upgrade stanza files online');
# Reread the info files and confirm the result
$oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza()));
$oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
$oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath());
$oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath());
$self->testResult(sub {$oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
$self->pgVersion())}, true, 'archive upgrade online corrects db');
$self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef,
@@ -409,10 +405,10 @@ sub run
my $strFullBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'update during backup',
{strOptionalParam => ' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=16384'});
{strOptionalParam => ' --buffer-size=16384'});
# Enabled async archiving
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_ARCHIVE_ASYNC) => 'y'}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'archive-async' => 'y'}});
# Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it
# down enough to make it evident that the async process is working.
@@ -444,7 +440,7 @@ sub run
}
$oHostDbStandby->restore(
'restore backup on replica', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'restore backup on replica', 'latest',
{rhRemapHash => \%oRemapHash, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY,
strOptionalParam =>
' --recovery-option="primary_conninfo=host=' . HOST_DB_MASTER .
@@ -470,19 +466,13 @@ sub run
{
my $strStandbyBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to reach master',
{bStandby => true,
iExpectedExitStatus => ERROR_DB_CONNECT,
strOptionalParam => '--' .
cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, cfgOptionIndexTotal(CFGOPT_PG_PATH))) . '=' . BOGUS});
{bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg8-host=' . BOGUS});
}
else
{
my $strStandbyBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to access at least one standby',
{bStandby => true,
iExpectedExitStatus => ERROR_DB_CONNECT,
strOptionalParam => '--' .
cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, cfgOptionIndexTotal(CFGOPT_PG_PATH))) . '=' . BOGUS});
{bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg8-host=' . BOGUS});
}
}
@@ -490,7 +480,7 @@ sub run
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby',
{bStandby => true,
iExpectedExitStatus => $oHostDbStandby->pgVersion() >= PG_VERSION_BACKUP_STANDBY ? undef : ERROR_CONFIG,
strOptionalParam => '--' . cfgOptionName(CFGOPT_REPO_RETENTION_FULL) . '=1'});
strOptionalParam => '--repo1-retention-full=1'});
if ($oHostDbStandby->pgVersion() >= PG_VERSION_BACKUP_STANDBY)
{
@@ -632,9 +622,7 @@ sub run
# Exercise --delta checksum option
my $strIncrBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'update during backup',
{strOptionalParam =>
'--' . cfgOptionName(CFGOPT_STOP_AUTO) . ' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=32768 --delta'});
CFGOPTVAL_BACKUP_TYPE_INCR, 'update during backup', {strOptionalParam => '--stop-auto --buffer-size=32768 --delta'});
# Ensure the check command runs properly with a tablespace unless there is a bogus host
if (!$oHostBackup->bogusHost())
@@ -702,9 +690,7 @@ sub run
if ($bTestLocal)
{
# Expect failure because postmaster.pid exists
$oHostDbMaster->restore(
'postmaster running', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
{iExpectedExitStatus => ERROR_POSTMASTER_RUNNING});
$oHostDbMaster->restore('postmaster running', 'latest', {iExpectedExitStatus => ERROR_POSTMASTER_RUNNING});
}
$oHostDbMaster->clusterStop();
@@ -712,8 +698,7 @@ sub run
if ($bTestLocal)
{
# Expect failure because db path is not empty
$oHostDbMaster->restore(
'path not empty', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
$oHostDbMaster->restore('path not empty', 'latest', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
}
# Drop and recreate db path
@@ -726,7 +711,7 @@ sub run
# Now the restore should work
$oHostDbMaster->restore(
undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
undef, 'latest',
{strOptionalParam => ($bTestLocal ? ' --db-include=test2 --db-include=test3' : '') . ' --buffer-size=16384'});
# Test that the first database has not been restored since --db-include did not include test1
@@ -784,8 +769,8 @@ sub run
{
# Backup info will have the catalog number
my $oBackupInfo = new pgBackRest::Common::Ini(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
{bLoad => false, strContent => ${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)}});
storageRepo(), $oHostBackup->repoBackupPath(FILE_BACKUP_INFO),
{bLoad => false, strContent => ${storageRepo()->get($oHostBackup->repoBackupPath(FILE_BACKUP_INFO))}});
# Construct the special path
$strTablespacePath .=
@@ -899,8 +884,7 @@ sub run
storageTest()->put($oHostDbMaster->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL);
}
$oHostDbMaster->restore(
undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE});
$oHostDbMaster->restore(undef, 'latest', {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE});
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $strXidMessage);
@@ -916,7 +900,7 @@ sub run
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
undef, 'latest',
{bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_TIME, strTarget => $strTimeTarget,
strTargetAction => $oHostDbMaster->pgVersion() >= PG_VERSION_91 ? 'promote' : undef,
strTargetTimeline => $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'current' : undef,
@@ -952,7 +936,7 @@ sub run
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
undef, 'latest',
{bDelta => true, bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_NAME, strTarget => $strNameTarget,
strTargetAction => 'promote',
strTargetTimeline => $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'current' : undef});
@@ -997,15 +981,13 @@ sub run
# Incr backup - make sure a --no-online backup fails
#-----------------------------------------------------------------------------------------------------------------------
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on --no-' . cfgOptionName(CFGOPT_ONLINE),
{iExpectedExitStatus => ERROR_POSTMASTER_RUNNING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on --no-online',
{iExpectedExitStatus => ERROR_POSTMASTER_RUNNING, strOptionalParam => '--no-online'});
# Incr backup - allow --no-online backup to succeed with --force
#-----------------------------------------------------------------------------------------------------------------------
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR,
'succeed on --no-' . cfgOptionName(CFGOPT_ONLINE) . ' with --' . cfgOptionName(CFGOPT_FORCE),
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
CFGOPTVAL_BACKUP_TYPE_INCR, 'succeed on --no-online with --force', {strOptionalParam => '--no-online --force'});
}
# Stanza-delete --force without access to pgbackrest on database host
@@ -1017,8 +999,8 @@ sub run
{
$oHostDbMaster->stop();
$oHostBackup->stop({strStanza => $self->stanza});
$oHostBackup->stanzaDelete("delete stanza with --force when pgbackrest on pg host not accessible",
{strOptionalParam => ' --' . cfgOptionName(CFGOPT_FORCE)});
$oHostBackup->stanzaDelete(
"delete stanza with --force when pgbackrest on pg host not accessible", {strOptionalParam => ' --force'});
$oHostDbMaster->start();
$oHostBackup->start();
}