db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-02-11 13:53:03 +02:00
Code Issues Releases Activity

Migrate integration tests to C.

Browse Source 
The Perl integration tests were migrated as faithfully as possible, but there was some cruft and a few unit tests that it did not make sense to migrate.

Also remove all Perl code made obsolete by this migration.

All unit, performance, and integration tests are now written in C but significant parts of the test harness remain to be migrated.
This commit is contained in:
David Steele 2024-03-06 11:00:09 +13:00
parent 7f1bb3a051
commit 794c577130
51 changed files with 3348 additions and 11225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/test.yml vendored
View File

@ -32,13 +32,13 @@ jobs:
- param: test --vm=d10 --param=no-performance --param=c-only
# All integration tests for 32-bit
- param: test --vm=d10 --param=module=mock --param=module=real
- param: test --vm=d10 --param=module=mock --param=module=integration
# Debian/Ubuntu documentation
- param: doc --vm=u20
# All integration tests
- param: test --vm=u22 --param=build-package --param=module=mock --param=module=real
- param: test --vm=u22 --param=build-package --param=module=mock --param=module=integration
# All unit tests with coverage, backtrace and alternate timezone
- param: test --vm=u22 --param=c-only --param=no-valgrind --param=tz=America/New_York
@ -53,7 +53,7 @@ jobs:
- param: doc --vm=rh8
# All integration tests
- param: test --vm=rh7 --param=module=mock --param=module=real
- param: test --vm=rh7 --param=module=mock --param=module=integration
steps:
- name: Checkout Code

6
doc/lib/pgBackRestDoc/Common/DocExecute.pm
View File

@ -16,11 +16,11 @@ use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestDoc::Common::DocManifest;
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Host;
use pgBackRestDoc::Common::HostGroup;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::Common::String;
@ -1050,7 +1050,7 @@ sub sectionChildProcess
$strOption =~ s/\{\[host\-repo\-path\]\}/${strHostRepoPath}/g;
}
my $oHost = new pgBackRestTest::Common::HostTest(
my $oHost = new pgBackRestDoc::Common::Host(
$$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage, $strHostUser,
defined($strMount) ? [$strMount] : undef, $strOption, $$hCacheKey{param}, $$hCacheKey{'update-hosts'});

4
test/lib/pgBackRestTest/Common/HostTest.pm → doc/lib/pgBackRestDoc/Common/Host.pm
View File

@ -1,7 +1,7 @@
####################################################################################################################################
# HostTest.pm - Encapsulate a docker host for testing
# HostTest.pm - Encapsulate a docker host
####################################################################################################################################
package pgBackRestTest::Common::HostTest;
package pgBackRestDoc::Common::Host;
####################################################################################################################################
# Perl includes

6
test/lib/pgBackRestTest/Common/HostGroupTest.pm → doc/lib/pgBackRestDoc/Common/HostGroup.pm
View File

@ -1,7 +1,7 @@
####################################################################################################################################
# HostGroupTest.pm - Encapsulate a group of docker containers for testing
# HostGroupTest.pm - Encapsulate a group of docker containers
####################################################################################################################################
package pgBackRestTest::Common::HostGroupTest;
package pgBackRestDoc::Common::HostGroup;
####################################################################################################################################
# Perl includes
@ -177,7 +177,7 @@ sub hostGroupGet
{
if (!defined($oHostGroup))
{
$oHostGroup = new pgBackRestTest::Common::HostGroupTest();
$oHostGroup = new pgBackRestDoc::Common::HostGroup();
}
return $oHostGroup;

20
src/build/common/exec.c
View File

@ -9,10 +9,11 @@ Execute Process Extensions
/**********************************************************************************************************************************/
static String *
execProcess(Exec *const this)
execProcess(Exec *const this, const ExecOneParam param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(EXEC, this);
FUNCTION_LOG_PARAM(INT, param.resultExpect);
FUNCTION_LOG_END();
String *const result = strNew();
@ -35,7 +36,7 @@ execProcess(Exec *const this)
// If the process exited normally but without a success status
if (WIFEXITED(processStatus))
{
if (WEXITSTATUS(processStatus) != 0)
if (WEXITSTATUS(processStatus) != param.resultExpect)
execCheckStatusError(this, processStatus, strTrim(result));
}
// Else if the process did not exit normally then it must have been a signal
@ -53,7 +54,8 @@ execOne(const String *const command, const ExecOneParam param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, command);
(void)param;
FUNCTION_LOG_PARAM(STRING, param.shell);
FUNCTION_LOG_PARAM(INT, param.resultExpect);
FUNCTION_LOG_END();
String *result = NULL;
@ -61,23 +63,23 @@ execOne(const String *const command, const ExecOneParam param)
MEM_CONTEXT_TEMP_BEGIN()
{
const StringList *const shellList = strLstNewSplitZ(param.shell != NULL ? param.shell : STRDEF("sh -c"), " ");
StringList *const param = strLstNew();
StringList *const paramList = strLstNew();
ASSERT(strLstSize(shellList) != 0);
for (unsigned int shellIdx = 1; shellIdx < strLstSize(shellList); shellIdx++)
strLstAdd(param, strLstGet(shellList, shellIdx));
strLstAdd(paramList, strLstGet(shellList, shellIdx));
strLstAddFmt(param, "%s 2>&1", strZ(command));
strLstAddZ(param, "2>&1");
strLstAddFmt(paramList, "%s 2>&1", strZ(command));
strLstAddZ(paramList, "2>&1");
Exec *const exec = execNew(strLstGet(shellList, 0), param, command, ioTimeoutMs());
Exec *const exec = execNew(strLstGet(shellList, 0), paramList, command, ioTimeoutMs());
execOpen(exec);
MEM_CONTEXT_PRIOR_BEGIN()
{
result = execProcess(exec);
result = execProcess(exec, param);
}
MEM_CONTEXT_PRIOR_END();
}

1
src/build/common/exec.h
View File

@ -17,6 +17,7 @@ typedef struct ExecOneParam
{
VAR_PARAM_HEADER;
const String *shell; // Shell command to use for exec (default is sh -c)
int resultExpect; // Expected result, if not 0
} ExecOneParam;
#define execOneP(command, ...) \

6
test/certificate/pgbackrest-test-server.cnf
View File

@ -37,6 +37,6 @@ DNS.4 = 127.0.0.1
IP.1 = 127.0.0.1
# Used in integration tests
DNS.5 = db-primary
DNS.6 = db-standby
DNS.7 = backup
DNS.5 = pg1
DNS.6 = pg2
DNS.7 = repo

33
test/certificate/pgbackrest-test-server.crt
View File

@ -1,8 +1,8 @@
-----BEGIN CERTIFICATE-----
MIIGAjCCA+qgAwIBAgIUW0gPWoZD5DqjIWIP3PliYA0IAOQwDQYJKoZIhvcNAQEL
MIIF8jCCA9qgAwIBAgIUJCya0E5vFzyH2AgiM3HSAHmpZ1QwDQYJKoZIhvcNAQEL
BQAwXDELMAkGA1UEBhMCVVMxDDAKBgNVBAgMA0FsbDEMMAoGA1UEBwwDQWxsMRMw
EQYDVQQKDApwZ0JhY2tSZXN0MRwwGgYDVQQDDBN0ZXN0LnBnYmFja3Jlc3Qub3Jn
MCAXDTIxMDgyNjEyMjkwM1oYDzIyOTUwNjEwMTIyOTAzWjB6MQswCQYDVQQGEwJV
MCAXDTI0MDMwNDAxMzgzMFoYDzIyOTcxMjE3MDEzODMwWjB6MQswCQYDVQQGEwJV
UzEMMAoGA1UECAwDQWxsMQwwCgYDVQQHDANBbGwxEzARBgNVBAoMCnBnQmFja1Jl
c3QxHDAaBgNVBAsME1VuaXQgVGVzdGluZyBEb21haW4xHDAaBgNVBAMME3Rlc3Qu
cGdiYWNrcmVzdC5vcmcwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDD
@ -16,20 +16,19 @@ dCdyXuSftGFx0JxvmDhl9qFGarv1BKgwO83j7sy3IREte1K21JaIHNBVWP+NwU0N
4Z4OMqnpnnnGiyi0xnfJVqOXghu5BLWl9MuOntZ0nnzLmFD7w795uNRgjE6jmRmF
FlX+PGqhHsZr0wZsBDsE9xO4i2l8aqJZx1hT5l3LIXC+lei/qo2gJi3nyePuz4UB
t53sTNEdrZndFUaRyq/aJfkR13J0eaoqKn5BRRHhw8tRef6S84e0kQ6ABYNRGHQN
V+GswPl1fV37114FTBnz2Bi/GSQSs8vWjw49HHKK5wIDAQABo4GbMIGYMAkGA1Ud
EwQCMAAwCwYDVR0PBAQDAgXgMH4GA1UdEQR3MHWCE3Rlc3QucGdiYWNrcmVzdC5v
V+GswPl1fV37114FTBnz2Bi/GSQSs8vWjw49HHKK5wIDAQABo4GLMIGIMAkGA1Ud
EwQCMAAwCwYDVR0PBAQDAgXgMG4GA1UdEQRnMGWCE3Rlc3QucGdiYWNrcmVzdC5v
cmeCFSoudGVzdC5wZ2JhY2tyZXN0Lm9yZ4IWKi50ZXN0Mi5wZ2JhY2tyZXN0Lm9y
Z4IJMTI3LjAuMC4xhwR/AAABggpkYi1wcmltYXJ5ggpkYi1zdGFuZGJ5ggZiYWNr
dXAwDQYJKoZIhvcNAQELBQADggIBAMbSq1/hjvQZJ2PFE/VVz9OcA7vlewq632eE
P5BalSJJgLVEsv1AxPx8VT08xfFQHQtEcCg/PFqT3RQ5yb1kHfa6glJkjYIdKQbn
lv9OVc/iutQwKPwk32QQjSgQFb/m0tXv9SlQ+gNTdkK4UKffXPj5rpgwaSiVwuLF
d+3TUpJihS48LLRC27kcL5Ur69/fu0ZD7xZSoCr/n8MUq4f9LwOhBqq+h64wM9cV
V79iPWmEJXoNAJrPYmK+XNhcro071c4m+HR4CCNikjxz/GUUf/NGHWT3pL0Ildku
X3dHmsNRVT/wLqi2v2oa6zr9FfVzjDAdCfnvTLOJ6H6dmofzQUFJBSWfhqGNDR8U
oblwirM2sjaOUjnkBS6Cb26yHSClStI+GZvS0KZfSVd2Qbe4YmtQMTNl/hdZGK3z
ZoqV++idVR+A0NQP8xR4VWqQdq0BR5eQOXDA4wtqvivqlIXpbJvqh1kBHPU9cAF+
g/t3Wa7EomwLazRaV9djLUpon6wGwScKJGzv+vyQSgXN1tQG9tLV4NCFUKDueUUZ
U/j1t64KF9hp5NU2A16zLp6V5GPIJhufXOYa66AFjV8c880eLd5YlkfzgyYwReOx
7vHkiLylbx2tc6aYUqdwjpMwnkxTsn52BBVxDvXToBIRdq/ea/LnZ/yhpnaac/Um
bJOTMee+
Z4IJMTI3LjAuMC4xhwR/AAABggNwZzGCA3BnMoIEcmVwbzANBgkqhkiG9w0BAQsF
AAOCAgEAX0MEXH6ANllRhdQz6neQ7SVG48Aj1lEAGeOhfpoNKzuyBcRjVw7+NyNN
IwlPKSSBDpnxaWQ5rCLtBtXod6yPMGKTRlFHwFFzfOps6nlRQjPsA848d6daLBvj
unpUQx4NFGPZJSs6z5z4BlT/+5mJVHC9qsrZBtkndYpRWo37xbVhRqP0+FSTbzrx
Gj2td3PoqQBgA/AmSKIpwagGzw7cSor1r4uEjkVxxyOMRbjuuASHMHUM7MtQV3YR
rz9UspvGfoKBdUkzMoqKRwxZWuh+uAoM+1GWXBjqlN6uAdQxpV2wZ75iRJp3Y4Bk
/CkXTLZ83lARGLqS/E5EFfg7Z9Bre2f5fHzV8C/h6WGpvCt/GlZqTx8fix/mMPT2
CFq+FcSmvF5JsIMxUnpvTw1hcTDNRPnOkFKnO1bjf9+jGCwzDUoGReYpb5veFSxh
IFkQ3oyw3/6v11aPstXSADTvRTFMyRklqu5NIHkMVQCPwQCAE3346KpEsT5HO2T7
X57sP05alMESjUv1sR260yYC3GUr0dbmf8gthVDhH1SsP3Drn/A3l70xGASnJA46
LCZwAJZ0KI6G/ok17lTZe9Hjwn2DkmVf1CKD7gXjmroYQI9O+etUtD/g+B2AISTG
SP60NakteOBWqmcSirV5npKh/SZR8Il5oaLS+3HerhgwvNDJ078=
-----END CERTIFICATE-----

4
test/ci.pl
View File

@ -183,7 +183,7 @@ eval
# Build list of packages that need to be installed
my $strPackage =
"make gcc ccache python3-pip git rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config" .
" uncrustify libssh2-1-dev";
" uncrustify libssh2-1-dev valgrind";
# Add lcov when testing coverage
if (vmCoverageC($strVm))
@ -194,7 +194,7 @@ eval
# Extra packages required when testing without containers
if ($strVm eq VM_NONE)
{
$strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev";
$strPackage .= " liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev";
}
# Else packages needed for integration tests on containers
else

12
test/define.yaml
View File

@ -608,7 +608,9 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: sftp
total: 19
harness: libSsh2
harness:
name: libSsh2
integration: false
harness:
name: fd
shim:
@ -635,7 +637,9 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: client
total: 1
harness: pq
harness:
name: pq
integration: false
coverage:
- postgres/client
@ -881,6 +885,7 @@ unit:
total: 12
harness:
name: backup
integration: false
shim:
command/backup/backup:
function:
@ -1004,13 +1009,14 @@ unit:
integration:
# ********************************************************************************************************************************
- name: real
- name: integration
db: true
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: all
total: 1
harness: host
# **********************************************************************************************************************************
# Performance tests

26
test/lib/pgBackRestTest/Common/CoverageTest.pm
View File

@ -26,7 +26,31 @@ use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::DefineTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::ListTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# testRunName
#
# Create module/test names by upper-casing the first letter and then inserting capitals after each -.
####################################################################################################################################
sub testRunName
{
my $strName = shift;
my $bInitCapFirst = shift;
$bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true;
my $bFirst = true;
my @stryName = split('\-', $strName);
$strName = undef;
foreach my $strPart (@stryName)
{
$strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart;
$bFirst = false;
}
return $strName;
}
####################################################################################################################################
# Generate an lcov configuration file

277
test/lib/pgBackRestTest/Common/FileTest.pm
View File

@ -1,277 +0,0 @@
####################################################################################################################################
# CommonTest.pm - Common globals used for testing
####################################################################################################################################
package pgBackRestTest::Common::FileTest;
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Cwd qw(abs_path cwd);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use File::Copy qw(move);
use File::Path qw(remove_tree);
use IO::Select;
use IPC::Open3;
use POSIX ':sys_wait_h';
use Symbol 'gensym';
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::Common::String;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageBase;
use pgBackRestTest::Common::VmTest;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostDbCommonTest;
use pgBackRestTest::Env::Host::HostDbTest;
use pgBackRestTest::Env::Host::HostS3Test;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# testLinkCreate
#
# Create a symlink
####################################################################################################################################
sub testLinkCreate
{
my $strLink = shift;
my $strDestination = shift;
# Create the file
symlink($strDestination, $strLink)
or confess "unable to link ${strLink} to ${strDestination}";
}
push(@EXPORT, qw(testLinkCreate));
####################################################################################################################################
# testPathMode
#
# Set mode of an existing path.
####################################################################################################################################
sub testPathMode
{
my $strPath = shift;
my $strMode = shift;
# Set the mode
chmod(oct($strMode), $strPath)
or confess 'unable to set mode ${strMode} for ${strPath}';
}
push(@EXPORT, qw(testPathMode));
####################################################################################################################################
# testPathRemove
#
# Remove a path and all subpaths.
####################################################################################################################################
sub testPathRemove
{
my $strPath = shift;
my $bSuppressError = shift;
executeTest('rm -rf ' . $strPath, {bSuppressError => $bSuppressError});
}
push(@EXPORT, qw(testPathRemove));
####################################################################################################################################
# testFileCreate
#
# Create a file specifying content, mode, and time.
####################################################################################################################################
sub testFileCreate
{
my $strFile = shift;
my $strContent = shift;
my $lTime = shift;
my $strMode = shift;
# Open the file and save strContent to it
my $hFile = shift;
open($hFile, '>', $strFile)
or confess "unable to open ${strFile} for writing";
if (defined($strContent) && $strContent ne '')
{
syswrite($hFile, $strContent)
or confess "unable to write to ${strFile}: $!";
}
close($hFile);
# Set the time
if (defined($lTime))
{
utime($lTime, $lTime, $strFile)
or confess 'unable to set time ${lTime} for ${strPath}';
}
# Set the mode
chmod(oct(defined($strMode) ? $strMode : '0600'), $strFile)
or confess 'unable to set mode ${strMode} for ${strFile}';
}
push(@EXPORT, qw(testFileCreate));
####################################################################################################################################
# testFileRemove
#
# Remove a file.
####################################################################################################################################
sub testFileRemove
{
my $strFile = shift;
unlink($strFile)
or confess "unable to remove ${strFile}: $!";
}
push(@EXPORT, qw(testFileRemove));
####################################################################################################################################
# forceStorageMode - force mode on a file or path
####################################################################################################################################
sub forceStorageMode
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strPath,
$strMode,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '::forceStorageMode', \@_,
{name => 'oStorage'},
{name => 'strPath'},
{name => 'strMode'},
{name => 'bRecurse', optional => true, default => false},
);
# Mode commands are ignored on object storage
if ($oStorage->type() ne STORAGE_OBJECT)
{
executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} ${strPath}");
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageMode));
####################################################################################################################################
# forceStorageMove - force move a directory or file
####################################################################################################################################
sub forceStorageMove
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strSourcePath,
$strDestinationPath,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->forceStorageMove', \@_,
{name => 'oStorage'},
{name => 'strSourcePath'},
{name => 'strDestinationPath'},
{name => 'bRecurse', optional => true, default => true},
);
# If object storage then use storage commands to remove
if ($oStorage->type() eq STORAGE_OBJECT)
{
if ($bRecurse)
{
my $rhManifest = $oStorage->manifest($strSourcePath);
foreach my $strName (sort(keys(%{$rhManifest})))
{
if ($rhManifest->{$strName}{type} eq 'f')
{
$oStorage->put(
"${strDestinationPath}/${strName}", ${$oStorage->get("${strSourcePath}/${strName}", {bRaw => true})},
{bRaw => true});
}
}
$oStorage->pathRemove($strSourcePath, {bRecurse => true});
}
else
{
$oStorage->put($strDestinationPath, ${$oStorage->get($strSourcePath, {bRaw => true})}, {bRaw => true});
$oStorage->remove($strSourcePath);
}
}
# Else remove using filesystem commands
else
{
executeTest("mv ${strSourcePath} ${strDestinationPath}");
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageMove));
####################################################################################################################################
# forceStorageRemove - force remove a file or path from storage
####################################################################################################################################
sub forceStorageRemove
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strPath,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '->forceStorageRemove', \@_,
{name => 'oStorage'},
{name => 'strPath'},
{name => 'bRecurse', optional => true, default => false},
);
# If object storage then use storage commands to remove
if ($oStorage->type() eq STORAGE_OBJECT)
{
$oStorage->pathRemove($strPath, {bRecurse => true});
}
else
{
executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $strPath);
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageRemove));
1;

77
test/lib/pgBackRestTest/Common/JobTest.pm
View File

@ -30,7 +30,6 @@ use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::DefineTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::ListTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::VmTest;
####################################################################################################################################
@ -115,7 +114,9 @@ sub new
$self->{iTry} = 0;
# Setup the path where unit test will be built
$self->{strUnitPath} = "$self->{strTestPath}/unit-$self->{iVmIdx}/$self->{oTest}->{&TEST_VM}";
$self->{strUnitPath} =
"$self->{strTestPath}/unit-$self->{iVmIdx}/" .
($self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? 'none' : $self->{oTest}->{&TEST_VM});
$self->{strDataPath} = "$self->{strTestPath}/data-$self->{iVmIdx}";
$self->{strRepoPath} = "$self->{strTestPath}/repo";
@ -246,58 +247,32 @@ sub run
if (!$self->{bDryRun} || $self->{bVmOut})
{
my $strCommand = undef; # Command to run test
my $bValgrind = $self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE;
my $strValgrindSuppress =
$self->{strRepoPath} . '/test/src/valgrind.suppress.' .
($self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? VM_NONE : $self->{oTest}->{&TEST_VM});
my $strVm = $self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? VM_NONE : $self->{oTest}->{&TEST_VM};
# If testing with C harness
if ($self->{oTest}->{&TEST_C})
{
# Create command
# ------------------------------------------------------------------------------------------------------------------
# Build filename for valgrind suppressions
my $strValgrindSuppress = $self->{strRepoPath} . '/test/src/valgrind.suppress.' . $self->{oTest}->{&TEST_VM};
$strCommand =
($self->{oTest}->{&TEST_VM} ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '" : '') .
" \\\n" .
$self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM} . '/test/src/test-pgbackrest' .
' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} .
" --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} .
' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') .
($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') .
($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') .
($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' .
$self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" .
# Allow stderr to be copied to stderr and stdout
"exec 3>&1 && \\\n" .
# Test with valgrind when requested
($self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE ?
'valgrind -q --gen-suppressions=all' .
($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') .
" --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') .
"$self->{strUnitPath}/build/test-unit 2>&1 1>&3 | tee /dev/stderr" .
($self->{oTest}->{&TEST_VM} ne VM_NONE ? "'" : '');
}
else
{
$strCommand =
($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') .
abs_path($0) .
" --test-path=${strVmTestPath}" .
" --vm=$self->{oTest}->{&TEST_VM}" .
" --vm-id=$self->{iVmIdx}" .
" --module=" . $self->{oTest}->{&TEST_MODULE} .
' --test=' . $self->{oTest}->{&TEST_NAME} .
$strCommandRunParam .
(defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') .
($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') .
($self->{strLogLevelTestFile} ne lc(TRACE) ? " --log-level-test-file=$self->{strLogLevelTestFile}" : '') .
my $strCommand =
($strVm ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '\\\n" : '') .
$self->{strTestPath} . "/build/${strVm}/test/src/test-pgbackrest" .
' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} .
" --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} .
' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') .
($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') .
' --psql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} .
($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') .
($self->{bDryRun} ? ' --dry-run' : '') .
($self->{bDryRun} ? ' --vm-out' : '') .
($self->{bNoCleanup} ? " --no-cleanup" : '');
}
(defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') .
($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' .
$self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" .
# Allow stderr to be copied to stderr and stdout
"exec 3>&1 && \\\n" .
# Test with valgrind when requested
($bValgrind ?
'valgrind -q --gen-suppressions=all' .
($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') .
" --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') .
"$self->{strUnitPath}/build/test-unit 2>&1 1>&3 | tee /dev/stderr" .
($strVm ne VM_NONE ? "'" : '');
my $oExec = new pgBackRestTest::Common::ExecuteTest(
$strCommand, {bSuppressError => true, bShowOutputAsync => $self->{bShowOutputAsync}});

426
test/lib/pgBackRestTest/Common/RunTest.pm
View File

@ -1,426 +0,0 @@
####################################################################################################################################
# RunTest.pm - All tests are inherited from this object
####################################################################################################################################
package pgBackRestTest::Common::RunTest;
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::Common::String;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::BuildTest;
use pgBackRestTest::Common::DefineTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::Storage;
use pgBackRestTest::Common::StoragePosix;
use pgBackRestTest::Common::VmTest;
use pgBackRestTest::Common::Wait;
####################################################################################################################################
# Constant to use when bogus data is required
####################################################################################################################################
use constant BOGUS => 'bogus';
push @EXPORT, qw(BOGUS);
####################################################################################################################################
# The current test run that is executing. Only a single run should ever occur in a process to prevent various cleanup issues from
# affecting the next run. Of course multiple subtests can be executed in a single run.
####################################################################################################################################
my $oTestRun;
my $oStorage;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');
# Initialize run counter
$self->{iRun} = 0;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# initModule
#
# Empty init sub in case the ancestor class does not declare one.
####################################################################################################################################
sub initModule {}
####################################################################################################################################
# initTest
#
# Empty init sub in case the ancestor class does not declare one.
####################################################################################################################################
sub initTest {}
####################################################################################################################################
# cleanTest
#
# Delete all files in test directory.
####################################################################################################################################
sub cleanTest
{
my $self = shift;
executeTest('rm -rf ' . $self->testPath() . '/*');
}
####################################################################################################################################
# cleanModule
#
# Empty final sub in case the ancestor class does not declare one.
####################################################################################################################################
sub cleanModule {}
####################################################################################################################################
# process
####################################################################################################################################
sub process
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strVm},
$self->{iVmId},
$self->{strBasePath},
$self->{strTestPath},
$self->{strBackRestExe},
$self->{strBackRestExeHelper},
$self->{strPgBinPath},
$self->{strPgVersion},
$self->{strModule},
$self->{strModuleTest},
$self->{iyModuleTestRun},
$self->{bOutput},
$self->{bDryRun},
$self->{bCleanup},
$self->{strLogLevelTestFile},
$self->{strPgUser},
$self->{strGroup},
) =
logDebugParam
(
__PACKAGE__ . '->process', \@_,
{name => 'strVm'},
{name => 'iVmId'},
{name => 'strBasePath'},
{name => 'strTestPath'},
{name => 'strBackRestExe'},
{name => 'strBackRestExeHelper'},
{name => 'strPgBinPath', required => false},
{name => 'strPgVersion', required => false},
{name => 'strModule'},
{name => 'strModuleTest'},
{name => 'iModuleTestRun', required => false},
{name => 'bOutput'},
{name => 'bDryRun'},
{name => 'bCleanup'},
{name => 'strLogLevelTestFile'},
{name => 'strPgUser'},
{name => 'strGroup'},
);
# Init will only be run on first test, clean/init on subsequent tests
$self->{bFirstTest} = true;
# Initialize test storage
$oStorage = new pgBackRestTest::Common::Storage(
$self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false}));
# Init, run, and clean the test(s)
$self->initModule();
$self->run();
$self->cleanModule();
# Make sure the correct number of tests ran
my $hModuleTest = testDefModuleTest($self->{strModule}, $self->{strModuleTest});
if ($hModuleTest->{&TESTDEF_TOTAL} != $self->runCurrent())
{
confess &log(ASSERT, "expected $hModuleTest->{&TESTDEF_TOTAL} tests to run but $self->{iRun} ran");
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# begin
####################################################################################################################################
sub begin
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDescription,
) =
logDebugParam
(
__PACKAGE__ . '->begin', \@_,
{name => 'strDescription'},
);
# Increment the run counter;
$self->{iRun}++;
# Return if this test should not be run
if (@{$self->{iyModuleTestRun}} != 0 && !grep(/^$self->{iRun}$/i, @{$self->{iyModuleTestRun}}))
{
return false;
}
# Output information about test to run
&log(INFO, 'run ' . sprintf('%03d', $self->runCurrent()) . ' - ' . $strDescription);
if ($self->isDryRun())
{
return false;
}
if (!$self->{bFirstTest})
{
$self->cleanTest();
}
$self->initTest();
$self->{bFirstTest} = false;
return true;
}
####################################################################################################################################
# testResult
####################################################################################################################################
sub testResult
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$fnSub,
$strExpected,
$strDescription,
$iWaitSeconds,
) =
logDebugParam
(
__PACKAGE__ . '::testResult', \@_,
{name => 'fnSub', trace => true},
{name => 'strExpected', required => false, trace => true},
{name => 'strDescription', trace => true},
{name => 'iWaitSeconds', optional => true, default => 0, trace => true},
);
&log(INFO, ' ' . $strDescription);
my $strActual;
my $bWarnValid = true;
my $oWait = waitInit($iWaitSeconds);
my $bDone = false;
# Clear the cache for this test
logFileCacheClear();
my @stryResult;
do
{
eval
{
@stryResult = ref($fnSub) eq 'CODE' ? $fnSub->() : $fnSub;
if (@stryResult <= 1)
{
$strActual = ${logDebugBuild($stryResult[0])};
}
else
{
$strActual = ${logDebugBuild(\@stryResult)};
}
return true;
}
or do
{
if (!isException(\$EVAL_ERROR))
{
confess "unexpected standard Perl exception" . (defined($EVAL_ERROR) ? ": ${EVAL_ERROR}" : '');
}
confess &logException($EVAL_ERROR);
};
if ($strActual ne (defined($strExpected) ? $strExpected : "[undef]"))
{
if (!waitMore($oWait))
{
confess
"expected:\n" . (defined($strExpected) ? "\"${strExpected}\"" : '[undef]') .
"\nbut actual was:\n" . (defined($strActual) ? "\"${strActual}\"" : '[undef]');
}
}
else
{
$bDone = true;
}
} while (!$bDone);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'result', value => \@stryResult, trace => true}
);
}
####################################################################################################################################
# testRunName
#
# Create module/test names by upper-casing the first letter and then inserting capitals after each -.
####################################################################################################################################
sub testRunName
{
my $strName = shift;
my $bInitCapFirst = shift;
$bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true;
my $bFirst = true;
my @stryName = split('\-', $strName);
$strName = undef;
foreach my $strPart (@stryName)
{
$strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart;
$bFirst = false;
}
return $strName;
}
push @EXPORT, qw(testRunName);
####################################################################################################################################
# testRun
####################################################################################################################################
sub testRun
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strModule,
$strModuleTest,
) =
logDebugParam
(
__PACKAGE__ . '::testRun', \@_,
{name => 'strModule', trace => true},
{name => 'strModuleTest', trace => true},
);
# Error if the test run is already defined - only one run per process is allowed
if (defined($oTestRun))
{
confess &log(ASSERT, 'a test run has already been created in this process');
}
my $strModuleName =
'pgBackRestTest::Module::' . testRunName($strModule) . '::' . testRunName($strModule) . testRunName($strModuleTest) .
'Test';
$oTestRun = eval("require ${strModuleName}; ${strModuleName}->import(); return new ${strModuleName}();")
or do {confess $EVAL_ERROR};
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oRun', value => $oTestRun, trace => true}
);
}
push @EXPORT, qw(testRun);
####################################################################################################################################
# testRunGet
####################################################################################################################################
sub testRunGet
{
return $oTestRun;
}
push @EXPORT, qw(testRunGet);
####################################################################################################################################
# storageTest - get the storage for the current test
####################################################################################################################################
sub storageTest
{
return $oStorage;
}
push(@EXPORT, qw(storageTest));
####################################################################################################################################
# Getters
####################################################################################################################################
sub archBits {return vmArchBits(shift->{strVm})}
sub backrestExe {return shift->{strBackRestExe}}
sub backrestExeHelper {return shift->{strBackRestExeHelper}}
sub basePath {return shift->{strBasePath}}
sub dataPath {return shift->basePath() . '/test/data'}
sub doCleanup {return shift->{bCleanup}}
sub logLevelTestFile {return shift->{strLogLevelTestFile}}
sub group {return shift->{strGroup}}
sub isDryRun {return shift->{bDryRun}}
sub module {return shift->{strModule}}
sub moduleTest {return shift->{strModuleTest}}
sub pgBinPath {return shift->{strPgBinPath}}
sub pgUser {return shift->{strPgUser}}
sub pgVersion {return shift->{strPgVersion}}
sub runCurrent {return shift->{iRun}}
sub stanza {return 'db'}
sub testPath {return shift->{strTestPath}}
sub vm {return shift->{strVm}}
sub vmId {return shift->{iVmId}}
1;

623
test/lib/pgBackRestTest/Common/StorageRepo.pm
View File

@ -1,623 +0,0 @@
####################################################################################################################################
# C Storage Interface
####################################################################################################################################
package pgBackRestTest::Common::StorageRepo;
use parent 'pgBackRestTest::Common::StorageBase';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Digest::SHA qw(sha1_hex);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use Fcntl qw(:mode);
use File::stat qw{lstat};
use JSON::PP;
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::Io::Handle;
use pgBackRestTest::Common::Io::Process;
use pgBackRestTest::Common::StorageBase;
####################################################################################################################################
# Temp file extension
####################################################################################################################################
use constant STORAGE_TEMP_EXT => PROJECT_EXE . '.tmp';
push @EXPORT, qw(STORAGE_TEMP_EXT);
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift;
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strCommand},
$self->{strType},
$self->{lBufferMax},
$self->{iTimeoutIo},
$self->{iRepo},
$self->{strDefaultPathMode},
$self->{strDefaultFileMode},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strCommand'},
{name => 'strType'},
{name => 'lBufferMax'},
{name => 'iTimeoutIo'},
{name => 'iRepo'},
{name => 'strDefaultPathMode', optional => true, default => '0750'},
{name => 'strDefaultFileMode', optional => true, default => '0640'},
);
# Create JSON object
$self->{oJSON} = JSON::PP->new()->allow_nonref();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# Escape characteres that have special meaning on the command line
####################################################################################################################################
sub escape
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strValue,
) =
logDebugParam
(
__PACKAGE__ . '->escape', \@_,
{name => 'strValue', trace => true},
);
$strValue =~ s/\\/\\\\/g;
$strValue =~ s/\</\\\</g;
$strValue =~ s/\>/\\\>/g;
$strValue =~ s/\!/\\\!/g;
$strValue =~ s/\*/\\\*/g;
$strValue =~ s/\(/\\\(/g;
$strValue =~ s/\)/\\\)/g;
$strValue =~ s/\&/\\\&/g;
$strValue =~ s/\'/\\\'/g;
$strValue =~ s/\;/\\\;/g;
$strValue =~ s/\?/\\\?/g;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strValue', value => $strValue},
);
}
####################################################################################################################################
# Execute command and return the output
####################################################################################################################################
sub exec
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCommand,
) =
logDebugParam
(
__PACKAGE__ . '->exec', \@_,
{name => 'strCommand'},
);
$strCommand = "$self->{strCommand} ${strCommand}";
my $oBuffer = new pgBackRestTest::Common::Io::Buffered(
new pgBackRestTest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax});
my $oProcess = new pgBackRestTest::Common::Io::Process($oBuffer, $strCommand);
my $tResult;
while (!$oBuffer->eof())
{
$oBuffer->read(\$tResult, $self->{lBufferMax}, false);
}
$oProcess->close();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'tResult', value => $tResult},
{name => 'iExitStatus', value => $oProcess->exitStatus()},
);
}
####################################################################################################################################
# Create storage
####################################################################################################################################
sub create
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->create');
$self->exec("--repo=$self->{iRepo} repo-create");
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# Check if file exists (not a path)
####################################################################################################################################
sub exists
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFileExp,
) =
logDebugParam
(
__PACKAGE__ . '->exists', \@_,
{name => 'strFileExp'},
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bExists', value => $self->info($strFileExp, {bIgnoreMissing => true})->{type} eq 'f'}
);
}
####################################################################################################################################
# Read a buffer from storage all at once
####################################################################################################################################
sub get
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFile,
$strCipherPass,
$bRaw,
) =
logDebugParam
(
__PACKAGE__ . '->get', \@_,
{name => 'xFile', required => false},
{name => 'strCipherPass', optional => true, redact => true},
{name => 'bRaw', optional => true, default => false},
);
# If openRead() was called first set values from that call
my $strFile = $xFile;
my $bIgnoreMissing = false;
if (ref($xFile))
{
$strFile = $xFile->{strFile};
$bIgnoreMissing = $xFile->{bIgnoreMissing};
$strCipherPass = $xFile->{strCipherPass};
}
# Check invalid params
if ($bRaw && defined($strCipherPass))
{
confess &log(ERROR, 'bRaw and strCipherPass cannot both be set');
}
# Get file
my ($tResult, $iExitStatus) = $self->exec(
(defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . ($bRaw ? ' --raw' : '') .
($bIgnoreMissing ? ' --ignore-missing' : '') . " --repo=$self->{iRepo} repo-get " . $self->escape($strFile));
# Error if missing an not ignored
if ($iExitStatus == 1 && !$bIgnoreMissing)
{
confess &log(ERROR, "unable to open '${strFile}'", ERROR_FILE_OPEN);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rtContent', value => $iExitStatus == 0 ? \$tResult : undef, trace => true},
);
}
####################################################################################################################################
# Get information for path/file
####################################################################################################################################
sub info
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathFileExp,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '->info', \@_,
{name => 'strPathFileExp'},
{name => 'bIgnoreMissing', optional => true, default => false},
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rhInfo', value => $self->manifest($strPathFileExp, {bRecurse => false})->{'.'}, trace => true}
);
}
####################################################################################################################################
# List all files/paths in path
####################################################################################################################################
sub list
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$strExpression,
$strSortOrder,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '->list', \@_,
{name => 'strPathExp', required => false},
{name => 'strExpression', optional => true},
{name => 'strSortOrder', optional => true, default => 'forward'},
{name => 'bIgnoreMissing', optional => true, default => false},
);
# Get file list
my $rstryFileList = [];
my $rhManifest = $self->manifest($strPathExp, {bRecurse => false});
foreach my $strKey ($strSortOrder eq 'reverse' ? sort {$b cmp $a} keys(%{$rhManifest}) : sort keys(%{$rhManifest}))
{
next if $strKey eq '.';
next if defined($strExpression) && $strKey !~ $strExpression;
push(@{$rstryFileList}, $strKey);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'stryFileList', value => $rstryFileList}
);
}
####################################################################################################################################
# Build path/file/link manifest starting with base path and including all subpaths
####################################################################################################################################
sub manifest
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->manifest', \@_,
{name => 'strPathExp'},
{name => 'bRecurse', optional => true, default => true},
);
my $rhManifest = $self->{oJSON}->decode(
$self->exec(
"--output=json" . ($bRecurse ? ' --recurse' : '') . " --repo=$self->{iRepo} repo-ls " . $self->escape($strPathExp)));
# Transform the manifest to the old format
foreach my $strKey (keys(%{$rhManifest}))
{
if ($rhManifest->{$strKey}{type} eq 'file')
{
$rhManifest->{$strKey}{type} = 'f';
if (defined($rhManifest->{$strKey}{time}))
{
$rhManifest->{$strKey}{modified_time} = $rhManifest->{$strKey}{time};
delete($rhManifest->{$strKey}{time});
}
}
elsif ($rhManifest->{$strKey}{type} eq 'path')
{
$rhManifest->{$strKey}{type} = 'd';
}
elsif ($rhManifest->{$strKey}{type} eq 'link')
{
$rhManifest->{$strKey}{type} = 'l';
}
elsif ($rhManifest->{$strKey}{type} eq 'special')
{
$rhManifest->{$strKey}{type} = 's';
}
else
{
confess "invalid file type '$rhManifest->{type}'";
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rhManifest', value => $rhManifest, trace => true}
);
}
####################################################################################################################################
# Open file for reading
####################################################################################################################################
sub openRead
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFile,
$bIgnoreMissing,
$strCipherPass,
) =
logDebugParam
(
__PACKAGE__ . '->openRead', \@_,
{name => 'strFile'},
{name => 'bIgnoreMissing', optional => true, default => false},
{name => 'strCipherPass', optional => true, redact => true},
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rhFileIo', value => {strFile => $strFile, bIgnoreMissing => $bIgnoreMissing, strCipherPass => $strCipherPass},
trace => true},
);
}
####################################################################################################################################
# Remove path and all files below it
####################################################################################################################################
sub pathRemove
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPath,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->pathRemove', \@_,
{name => 'strPath'},
{name => 'bRecurse', optional => true, default => false},
);
$self->exec("--repo=$self->{iRepo} repo-rm " . ($bRecurse ? '--recurse ' : '') . $self->escape($strPath));
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# put - writes a buffer out to storage all at once
####################################################################################################################################
sub put
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFile,
$tContent,
$strCipherPass,
$bRaw,
) =
logDebugParam
(
__PACKAGE__ . '->put', \@_,
{name => 'strFile'},
{name => 'tContent', required => false},
{name => 'strCipherPass', optional => true, redact => true},
{name => 'bRaw', optional => true, default => false},
);
# Check invalid params
if ($bRaw && defined($strCipherPass))
{
confess &log(ERROR, 'bRaw and strCipherPass cannot both be set');
}
# Put file
my $strCommand =
"$self->{strCommand}" . (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') .
($bRaw ? ' --raw' : '') . " --repo=$self->{iRepo} repo-put " . $self->escape($strFile);
my $oBuffer = new pgBackRestTest::Common::Io::Buffered(
new pgBackRestTest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax});
my $oProcess = new pgBackRestTest::Common::Io::Process($oBuffer, $strCommand);
if (defined($tContent))
{
$oBuffer->write(\$tContent);
}
close($oBuffer->handleWrite());
my $tResult;
while (!$oBuffer->eof())
{
$oBuffer->read(\$tResult, $self->{lBufferMax}, false);
}
close($oBuffer->handleRead());
$oProcess->close();
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# Remove file
####################################################################################################################################
sub remove
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFile,
) =
logDebugParam
(
__PACKAGE__ . '->remove', \@_,
{name => 'xFileExp'},
);
$self->exec("--repo=$self->{iRepo} repo-rm " . $self->escape($strFile));
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# Cache storage so it can be retrieved quickly
####################################################################################################################################
my $oRepoStorage;
####################################################################################################################################
# storageRepoCommandSet
####################################################################################################################################
my $strStorageRepoCommand;
my $strStorageRepoType;
sub storageRepoCommandSet
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCommand,
$strStorageType,
) =
logDebugParam
(
__PACKAGE__ . '::storageRepoCommandSet', \@_,
{name => 'strCommand'},
{name => 'strStorageType'},
);
$strStorageRepoCommand = $strCommand;
$strStorageRepoType = $strStorageType;
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push @EXPORT, qw(storageRepoCommandSet);
####################################################################################################################################
# storageRepo - get repository storage
####################################################################################################################################
sub storageRepo
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
$iRepo,
) =
logDebugParam
(
__PACKAGE__ . '::storageRepo', \@_,
{name => 'strStanza', optional => true, trace => true},
{name => 'iRepo', optional => true, default => 1, trace => true},
);
# Create storage if not defined
if (!defined($oRepoStorage->{$iRepo}))
{
$oRepoStorage->{$iRepo} = new pgBackRestTest::Common::StorageRepo(
$strStorageRepoCommand, $strStorageRepoType, 64 * 1024, 60, $iRepo);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oStorageRepo', value => $oRepoStorage->{$iRepo}, trace => true},
);
}
push @EXPORT, qw(storageRepo);
####################################################################################################################################
# Getters
####################################################################################################################################
sub capability {shift->type() eq STORAGE_POSIX}
sub type {shift->{strType}}
1;

478
test/lib/pgBackRestTest/Env/ArchiveInfo.pm
View File

@ -1,478 +0,0 @@
####################################################################################################################################
# ARCHIVE INFO MODULE
#
# The archive.info file is created when archiving begins. It is located under the stanza directory. The file contains information
# regarding the stanza database version, database WAL segment system id and other information to ensure that archiving is being
# performed on the proper database.
####################################################################################################################################
package pgBackRestTest::Env::ArchiveInfo;
use parent 'pgBackRestDoc::Common::Ini';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname basename);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::StorageBase;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Env::InfoCommon;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# File/path constants
####################################################################################################################################
use constant ARCHIVE_INFO_FILE => 'archive.info';
push @EXPORT, qw(ARCHIVE_INFO_FILE);
####################################################################################################################################
# RegEx constants
####################################################################################################################################
use constant REGEX_ARCHIVE_DIR_DB_VERSION => '^[0-9]+(\.[0-9]+)*-[0-9]+$';
push @EXPORT, qw(REGEX_ARCHIVE_DIR_DB_VERSION);
use constant REGEX_ARCHIVE_DIR_WAL => '^[0-F]{16}$';
push @EXPORT, qw(REGEX_ARCHIVE_DIR_WAL);
####################################################################################################################################
# WAL segment size
####################################################################################################################################
use constant PG_WAL_SEGMENT_SIZE => 16777216;
push @EXPORT, qw(PG_WAL_SEGMENT_SIZE);
####################################################################################################################################
# Archive info constants
####################################################################################################################################
use constant INFO_ARCHIVE_SECTION_DB => INFO_BACKUP_SECTION_DB;
push @EXPORT, qw(INFO_ARCHIVE_SECTION_DB);
use constant INFO_ARCHIVE_SECTION_DB_HISTORY => INFO_BACKUP_SECTION_DB_HISTORY;
push @EXPORT, qw(INFO_ARCHIVE_SECTION_DB_HISTORY);
use constant INFO_ARCHIVE_KEY_DB_VERSION => MANIFEST_KEY_DB_VERSION;
push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_VERSION);
use constant INFO_ARCHIVE_KEY_DB_ID => MANIFEST_KEY_DB_ID;
push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_ID);
use constant INFO_ARCHIVE_KEY_DB_SYSTEM_ID => MANIFEST_KEY_SYSTEM_ID;
push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_SYSTEM_ID);
####################################################################################################################################
# Global variables
####################################################################################################################################
my $strArchiveInfoMissingMsg =
ARCHIVE_INFO_FILE . " does not exist but is required to push/get WAL segments\n" .
"HINT: is archive_command configured in postgresql.conf?\n" .
"HINT: has a stanza-create been performed?\n" .
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.";
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strArchiveClusterPath, # Archive cluster path
$bRequired, # Is archive info required?
$bLoad, # Should the file attempt to be loaded?
$bIgnoreMissing, # Don't error on missing files
$strCipherPassSub, # Passphrase to encrypt the subsequent archive files if repo is encrypted
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strArchiveClusterPath'},
{name => 'bRequired', default => true},
{name => 'bLoad', optional => true, default => true},
{name => 'bIgnoreMissing', optional => true, default => false},
{name => 'strCipherPassSub', optional => true},
);
# Build the archive info path/file name
my $strArchiveInfoFile = "${strArchiveClusterPath}/" . ARCHIVE_INFO_FILE;
my $self = {};
my $iResult = 0;
my $strResultMessage;
# Init object and store variables
eval
{
$self = $class->SUPER::new(
storageRepo(), $strArchiveInfoFile,
{bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub});
return true;
}
or do
{
# Capture error information
$iResult = exceptionCode($EVAL_ERROR);
$strResultMessage = exceptionMessage($EVAL_ERROR);
};
if ($iResult != 0)
{
# If the file does not exist but is required to exist, then error
# The archive info is only allowed not to exist when running a stanza-create on a new install
if ($iResult == ERROR_FILE_MISSING)
{
if ($bRequired)
{
confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING);
}
}
elsif ($iResult == ERROR_CRYPTO && $strResultMessage =~ "^unable to flush")
{
confess &log(ERROR, "unable to parse '$strArchiveInfoFile'\nHINT: is or was the repo encrypted?", $iResult);
}
else
{
confess $EVAL_ERROR;
}
}
$self->{strArchiveClusterPath} = $strArchiveClusterPath;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# check
#
# Check archive info file and make sure it is compatible with the current version of the database for the stanza. If the file does
# not exist an error will occur.
####################################################################################################################################
sub check
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$ullDbSysId,
$bRequired,
) =
logDebugParam
(
__PACKAGE__ . '->check', \@_,
{name => 'strDbVersion'},
{name => 'ullDbSysId'},
{name => 'bRequired', default => true},
);
# ??? remove bRequired after stanza-upgrade
if ($bRequired)
{
# Confirm the info file exists with the DB section
$self->confirmExists();
}
my $strError = undef;
if (!$self->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $strDbVersion))
{
$strError = "WAL segment version ${strDbVersion} does not match archive version " .
$self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION);
}
if (!$self->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, $ullDbSysId))
{
$strError = (defined($strError) ? ($strError . "\n") : "") .
"WAL segment system-id ${ullDbSysId} does not match archive system-id " .
$self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID);
}
if (defined($strError))
{
confess &log(ERROR, "${strError}\nHINT: are you archiving to the correct stanza?", ERROR_ARCHIVE_MISMATCH);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveId', value => $self->archiveId()}
);
}
####################################################################################################################################
# archiveId
#
# Get the archive id which is a combination of the DB version and the db-id setting (e.g. 9.4-1)
####################################################################################################################################
sub archiveId
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$ullDbSysId,
) = logDebugParam
(
__PACKAGE__ . '->archiveId', \@_,
{name => 'strDbVersion', optional => true},
{name => 'ullDbSysId', optional => true},
);
my $strArchiveId = undef;
# If neither optional version and system-id are passed then set the archive id to the current one
if (!defined($strDbVersion) && !defined($ullDbSysId))
{
$strArchiveId = $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION) . "-" .
$self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID);
}
# If both the optional version and system-id are passed
elsif (defined($strDbVersion) && defined($ullDbSysId))
{
# Get the newest archiveId for the version/system-id passed
$strArchiveId = ($self->archiveIdList($strDbVersion, $ullDbSysId))[0];
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveId', value => $strArchiveId}
);
}
####################################################################################################################################
# archiveIdList
#
# Get a sorted list of the archive ids for the db-version and db-system-id passed.
####################################################################################################################################
sub archiveIdList
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$ullDbSysId,
) = logDebugParam
(
__PACKAGE__ . '->archiveIdList', \@_,
{name => 'strDbVersion'},
{name => 'ullDbSysId'},
);
my @stryArchiveId;
# Get the version and system-id for all known databases
my $hDbList = $self->dbHistoryList();
foreach my $iDbHistoryId (sort {$a <=> $b} keys %$hDbList)
{
# If the version and system-id match then construct the archive id so that the constructed array has the newest match first
if (($hDbList->{$iDbHistoryId}{&INFO_DB_VERSION} eq $strDbVersion) &&
($hDbList->{$iDbHistoryId}{&INFO_SYSTEM_ID} eq $ullDbSysId))
{
unshift(@stryArchiveId, $strDbVersion . "-" . $iDbHistoryId);
}
}
# If the archive id has still not been found, then error
if (@stryArchiveId == 0)
{
confess &log(
ERROR, "unable to retrieve the archive id for database version '$strDbVersion' and system-id '$ullDbSysId'",
ERROR_ARCHIVE_MISMATCH);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'stryArchiveId', value => \@stryArchiveId}
);
}
####################################################################################################################################
# create
#
# Creates the archive.info file. WARNING - this function should only be called from stanza-create or tests.
####################################################################################################################################
sub create
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$ullDbSysId,
$bSave,
) =
logDebugParam
(
__PACKAGE__ . '->create', \@_,
{name => 'strDbVersion'},
{name => 'ullDbSysId'},
{name => 'bSave', default => true},
);
# Fill db section and db history section
$self->dbSectionSet($strDbVersion, $ullDbSysId, $self->dbHistoryIdGet(false));
if ($bSave)
{
$self->save();
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# dbHistoryIdGet
#
# Get the db history ID
####################################################################################################################################
sub dbHistoryIdGet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$bFileRequired,
) =
logDebugParam
(
__PACKAGE__ . '->dbHistoryIdGet', \@_,
{name => 'bFileRequired', default => true},
);
# Confirm the info file exists if it is required
if ($bFileRequired)
{
$self->confirmExists();
}
# If the DB section does not exist, initialize the history to one, else return the latest ID
my $iDbHistoryId = (!$self->test(INFO_ARCHIVE_SECTION_DB))
? 1 : $self->numericGet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iDbHistoryId', value => $iDbHistoryId}
);
}
####################################################################################################################################
# dbHistoryList
#
# Get the data from the db history section.
####################################################################################################################################
sub dbHistoryList
{
my $self = shift;
my
(
$strOperation,
) = logDebugParam
(
__PACKAGE__ . '->dbHistoryList',
);
my %hDbHash;
foreach my $iHistoryId ($self->keys(INFO_ARCHIVE_SECTION_DB_HISTORY))
{
$hDbHash{$iHistoryId}{&INFO_DB_VERSION} =
$self->get(INFO_ARCHIVE_SECTION_DB_HISTORY, $iHistoryId, INFO_ARCHIVE_KEY_DB_VERSION);
$hDbHash{$iHistoryId}{&INFO_SYSTEM_ID} =
$self->get(INFO_ARCHIVE_SECTION_DB_HISTORY, $iHistoryId, INFO_ARCHIVE_KEY_DB_ID);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hDbHash', value => \%hDbHash}
);
}
####################################################################################################################################
# dbSectionSet
#
# Set the db and db:history sections.
####################################################################################################################################
sub dbSectionSet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$ullDbSysId,
$iDbHistoryId,
) =
logDebugParam
(
__PACKAGE__ . '->dbSectionSet', \@_,
{name => 'strDbVersion', trace => true},
{name => 'ullDbSysId', trace => true},
{name => 'iDbHistoryId', trace => true}
);
# Fill db section
$self->numericSet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, $ullDbSysId);
# Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one
$self->set(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $strDbVersion . '');
$self->numericSet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID, undef, $iDbHistoryId);
# Fill db history
$self->numericSet(INFO_ARCHIVE_SECTION_DB_HISTORY, $iDbHistoryId, INFO_ARCHIVE_KEY_DB_ID, $ullDbSysId);
# Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one
$self->set(INFO_ARCHIVE_SECTION_DB_HISTORY, $iDbHistoryId, INFO_ARCHIVE_KEY_DB_VERSION, $strDbVersion . '');
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# confirmExists
#
# Ensure that the archive.info file and the db section exist.
####################################################################################################################################
sub confirmExists
{
my $self = shift;
# Confirm the file exists and the DB section is filled out
if (!$self->test(INFO_ARCHIVE_SECTION_DB) || !$self->{bExists})
{
confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING);
}
}
1;

898
test/lib/pgBackRestTest/Env/BackupInfo.pm
View File

@ -1,898 +0,0 @@
####################################################################################################################################
# BACKUP INFO MODULE
####################################################################################################################################
package pgBackRestTest::Env::BackupInfo;
use parent 'pgBackRestDoc::Common::Ini';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname basename);
use File::stat;
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Env::ArchiveInfo;
use pgBackRestTest::Env::InfoCommon;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# Backup type constants
####################################################################################################################################
use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL);
use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF);
use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR);
####################################################################################################################################
# File/path constants
####################################################################################################################################
use constant FILE_BACKUP_INFO => 'backup.info';
push @EXPORT, qw(FILE_BACKUP_INFO);
####################################################################################################################################
# Backup info constants
####################################################################################################################################
use constant INFO_BACKUP_SECTION_BACKUP => MANIFEST_SECTION_BACKUP;
push @EXPORT, qw(INFO_BACKUP_SECTION_BACKUP);
use constant INFO_BACKUP_SECTION_BACKUP_CURRENT => INFO_BACKUP_SECTION_BACKUP . ':current';
push @EXPORT, qw(INFO_BACKUP_SECTION_BACKUP_CURRENT);
use constant INFO_BACKUP_KEY_ARCHIVE_CHECK => MANIFEST_KEY_ARCHIVE_CHECK;
push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_CHECK);
use constant INFO_BACKUP_KEY_ARCHIVE_COPY => MANIFEST_KEY_ARCHIVE_COPY;
push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_COPY);
use constant INFO_BACKUP_KEY_ARCHIVE_START => MANIFEST_KEY_ARCHIVE_START;
push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_START);
use constant INFO_BACKUP_KEY_ARCHIVE_STOP => MANIFEST_KEY_ARCHIVE_STOP;
push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_STOP);
use constant INFO_BACKUP_KEY_BACKUP_STANDBY => MANIFEST_KEY_BACKUP_STANDBY;
push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_STANDBY);
use constant INFO_BACKUP_KEY_BACKUP_REPO_SIZE => 'backup-info-repo-size';
push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_REPO_SIZE);
use constant INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA => 'backup-info-repo-size-delta';
push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA);
use constant INFO_BACKUP_KEY_BACKUP_SIZE => 'backup-info-size';
push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_SIZE);
use constant INFO_BACKUP_KEY_BACKUP_SIZE_DELTA => 'backup-info-size-delta';
push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_SIZE_DELTA);
use constant INFO_BACKUP_KEY_CATALOG => MANIFEST_KEY_CATALOG;
push @EXPORT, qw(INFO_BACKUP_KEY_CATALOG);
use constant INFO_BACKUP_KEY_CONTROL => MANIFEST_KEY_CONTROL;
push @EXPORT, qw(INFO_BACKUP_KEY_CONTROL);
use constant INFO_BACKUP_KEY_COMPRESS => MANIFEST_KEY_COMPRESS;
push @EXPORT, qw(INFO_BACKUP_KEY_COMPRESS);
use constant INFO_BACKUP_KEY_CHECKSUM_PAGE => MANIFEST_KEY_CHECKSUM_PAGE;
push @EXPORT, qw(INFO_BACKUP_KEY_CHECKSUM_PAGE);
use constant INFO_BACKUP_KEY_DB_VERSION => MANIFEST_KEY_DB_VERSION;
push @EXPORT, qw(INFO_BACKUP_KEY_DB_VERSION);
use constant INFO_BACKUP_KEY_FORMAT => INI_KEY_FORMAT;
push @EXPORT, qw(INFO_BACKUP_KEY_FORMAT);
use constant INFO_BACKUP_KEY_HARDLINK => MANIFEST_KEY_HARDLINK;
push @EXPORT, qw(INFO_BACKUP_KEY_HARDLINK);
use constant INFO_BACKUP_KEY_HISTORY_ID => MANIFEST_KEY_DB_ID;
push @EXPORT, qw(INFO_BACKUP_KEY_HISTORY_ID);
use constant INFO_BACKUP_KEY_LABEL => MANIFEST_KEY_LABEL;
push @EXPORT, qw(INFO_BACKUP_KEY_LABEL);
use constant INFO_BACKUP_KEY_PRIOR => MANIFEST_KEY_PRIOR;
push @EXPORT, qw(INFO_BACKUP_KEY_PRIOR);
use constant INFO_BACKUP_KEY_REFERENCE => 'backup-reference';
push @EXPORT, qw(INFO_BACKUP_KEY_REFERENCE);
use constant INFO_BACKUP_KEY_ONLINE => MANIFEST_KEY_ONLINE;
push @EXPORT, qw(INFO_BACKUP_KEY_ONLINE);
use constant INFO_BACKUP_KEY_SYSTEM_ID => MANIFEST_KEY_SYSTEM_ID;
push @EXPORT, qw(INFO_BACKUP_KEY_SYSTEM_ID);
use constant INFO_BACKUP_KEY_TIMESTAMP_START => MANIFEST_KEY_TIMESTAMP_START;
push @EXPORT, qw(INFO_BACKUP_KEY_TIMESTAMP_START);
use constant INFO_BACKUP_KEY_TIMESTAMP_STOP => MANIFEST_KEY_TIMESTAMP_STOP;
push @EXPORT, qw(INFO_BACKUP_KEY_TIMESTAMP_STOP);
use constant INFO_BACKUP_KEY_TYPE => MANIFEST_KEY_TYPE;
push @EXPORT, qw(INFO_BACKUP_KEY_TYPE);
use constant INFO_BACKUP_KEY_VERSION => INI_KEY_VERSION;
push @EXPORT, qw(INFO_BACKUP_KEY_VERSION);
####################################################################################################################################
# Global variables
####################################################################################################################################
my $strBackupInfoMissingMsg =
FILE_BACKUP_INFO . " does not exist and is required to perform a backup.\n" .
"HINT: has a stanza-create been performed?";
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strBackupClusterPath,
$bRequired,
$oStorage,
$bLoad, # Should the file attemp to be loaded?
$bIgnoreMissing, # Don't error on missing files
$strCipherPassSub, # Generated passphrase to encrypt manifest files if the repo is encrypted
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strBackupClusterPath'},
{name => 'bRequired', default => true},
{name => 'oStorage', optional => true, default => storageRepo()},
{name => 'bLoad', optional => true, default => true},
{name => 'bIgnoreMissing', optional => true, default => false},
{name => 'strCipherPassSub', optional => true},
);
# Build the backup info path/file name
my $strBackupInfoFile = "${strBackupClusterPath}/" . FILE_BACKUP_INFO;
my $self = {};
my $iResult = 0;
my $strResultMessage;
# Init object and store variables
eval
{
$self = $class->SUPER::new(
$oStorage, $strBackupInfoFile,
{bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub});
return true;
}
or do
{
# Capture error information
$iResult = exceptionCode($EVAL_ERROR);
$strResultMessage = exceptionMessage($EVAL_ERROR);
};
if ($iResult != 0)
{
# If the backup info file does not exist and is required, then throw an error
# The backup info is only allowed not to exist when running a stanza-create on a new install
if ($iResult == ERROR_FILE_MISSING)
{
if ($bRequired)
{
confess &log(ERROR, "${strBackupClusterPath}/$strBackupInfoMissingMsg", ERROR_FILE_MISSING);
}
}
elsif ($iResult == ERROR_CRYPTO && $strResultMessage =~ "^unable to flush")
{
confess &log(ERROR, "unable to parse '$strBackupInfoFile'\nHINT: is or was the repo encrypted?", $iResult);
}
else
{
confess $EVAL_ERROR;
}
}
$self->{strBackupClusterPath} = $strBackupClusterPath;
$self->{oStorage} = $oStorage;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# check
#
# Check db info and make sure it matches what is already in the repository. Return the db-id if everything matches.
####################################################################################################################################
sub check
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$iControlVersion,
$iCatalogVersion,
$ullDbSysId,
$bRequired,
) =
logDebugParam
(
__PACKAGE__ . '->check', \@_,
{name => 'strDbVersion', trace => true},
{name => 'iControlVersion', trace => true},
{name => 'iCatalogVersion', trace => true},
{name => 'ullDbSysId', trace => true},
{name => 'bRequired', default => true},
);
# Confirm the info file exists with the DB section
if ($bRequired)
{
$self->confirmExists();
}
if (!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID, undef, $ullDbSysId) ||
!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, $strDbVersion))
{
confess &log(ERROR, "database version = ${strDbVersion}, system-id ${ullDbSysId} does not match backup version = " .
$self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION) . ", system-id = " .
$self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID) . "\n" .
"HINT: is this the correct stanza?", ERROR_BACKUP_MISMATCH);
}
if (!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG, undef, $iCatalogVersion) ||
!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL, undef, $iControlVersion))
{
confess &log(ERROR, "database control-version = ${iControlVersion}, catalog-version ${iCatalogVersion}" .
" does not match backup control-version = " .
$self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL) . ", catalog-version = " .
$self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG) . "\n" .
"HINT: this may be a symptom of database or repository corruption!", ERROR_BACKUP_MISMATCH);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iDbHistoryId', value => $self->numericGet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID)}
);
}
####################################################################################################################################
# add
#
# Add a backup to the info file.
####################################################################################################################################
sub add
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oBackupManifest,
$bSave,
$bRequired,
) =
logDebugParam
(
__PACKAGE__ . '->add', \@_,
{name => 'oBackupManifest', trace => true},
{name => 'bSave', default => true, trace => true},
{name => 'bRequired', default => true, trace => true},
);
# Confirm the info file exists with the DB section
if ($bRequired)
{
$self->confirmExists();
}
# Get the backup label
my $strBackupLabel = $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL);
# Calculate backup sizes and references
my $lBackupSize = 0;
my $lBackupSizeDelta = 0;
my $lBackupRepoSize = 0;
my $lBackupRepoSizeDelta = 0;
my $oReferenceHash = undef;
foreach my $strFileKey ($oBackupManifest->keys(MANIFEST_SECTION_TARGET_FILE))
{
my $lFileSize =
$oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_SIZE);
my $lRepoSize =
$oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false, $lFileSize);
my $strFileReference =
$oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE, false);
# Temporary until compressed size is back in
$lBackupSize += $lFileSize;
$lBackupRepoSize += $lRepoSize;
if (defined($strFileReference))
{
$$oReferenceHash{$strFileReference} = true;
}
else
{
$lBackupSizeDelta += $lFileSize;
$lBackupRepoSizeDelta += $lRepoSize;
}
}
# Set backup size info
$self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_SIZE, $lBackupSize);
$self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_SIZE_DELTA, $lBackupSizeDelta);
$self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_REPO_SIZE, $lBackupRepoSize);
$self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA,
$lBackupRepoSizeDelta);
$self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_CHECK,
$oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK));
$self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_COPY,
$oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY));
$self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_START,
$oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, false));
$self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_STOP,
$oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, false));
$self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_STANDBY,
$oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY));
$self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_CHECKSUM_PAGE,
$oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE));
$self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_COMPRESS,
$oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS));
$self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_FORMAT,
$oBackupManifest->numericGet(INI_SECTION_BACKREST, INI_KEY_FORMAT));
$self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HARDLINK,
$oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK));
$self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ONLINE,
$oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE));
$self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TIMESTAMP_START,
$oBackupManifest->numericGet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START));
$self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TIMESTAMP_STOP,
$oBackupManifest->numericGet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP));
$self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TYPE,
$oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE));
$self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_VERSION,
$oBackupManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION));
if ($bRequired)
{
$self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID,
$self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID));
}
# If we are reconstructing, then the history id must be taken from the manifest
else
{
$self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID,
$oBackupManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID));
}
if (!$oBackupManifest->test(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, CFGOPTVAL_BACKUP_TYPE_FULL))
{
my @stryReference = sort(keys(%$oReferenceHash));
$self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_PRIOR,
$oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR));
$self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_REFERENCE,
\@stryReference);
}
if ($bSave)
{
$self->save();
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# current
#
# Test if a backup is current.
####################################################################################################################################
sub current
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strBackup
) =
logDebugParam
(
__PACKAGE__ . '->current', \@_,
{name => 'strBackup'}
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bTest', value => $self->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup)}
);
}
####################################################################################################################################
# list
#
# Get backup keys.
####################################################################################################################################
sub list
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFilter,
$strOrder
) =
logDebugParam
(
__PACKAGE__ . '->list', \@_,
{name => 'strFilter', required => false},
{name => 'strOrder', default => 'forward'}
);
# List of backups
my @stryBackup;
# Iterate through the backups and filter
for my $strBackup ($self->keys(INFO_BACKUP_SECTION_BACKUP_CURRENT))
{
if (!defined($strFilter) || $strBackup =~ $strFilter)
{
if ($strOrder eq 'reverse')
{
unshift(@stryBackup, $strBackup)
}
else
{
push(@stryBackup, $strBackup)
}
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'stryBackup', value => \@stryBackup}
);
}
####################################################################################################################################
# backupArchiveDbHistoryId
#
# Gets the backup.info db-id for the archiveId passed.
####################################################################################################################################
sub backupArchiveDbHistoryId
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strArchiveId,
$strPathBackupArchive,
) =
logDebugParam
(
__PACKAGE__ . '->backupArchiveDbHistoryId', \@_,
{name => 'strArchiveId'},
{name => 'strPathBackupArchive'},
);
# List of backups associated with the db-id provided
my @stryArchiveBackup;
# Build the db list from the history in the backup info and archive info file
my $oArchiveInfo = new pgBackRestTest::Env::ArchiveInfo($strPathBackupArchive, true);
my $hDbListArchive = $oArchiveInfo->dbHistoryList();
my $hDbListBackup = $self->dbHistoryList();
my $iDbHistoryId = undef;
# Get the db-version and db-id (history id) from the archiveId
my ($strDbVersionArchive, $iDbIdArchive) = split("-", $strArchiveId);
# Get the DB system ID to map back to the backup info if it exists in the archive info file
if (exists($hDbListArchive->{$iDbIdArchive}))
{
my $ullDbSysIdArchive = $$hDbListArchive{$iDbIdArchive}{&INFO_SYSTEM_ID};
# Get the db-id from backup info history that corresponds to the archive db-version and db-system-id
# Sort from newest (highest db-id) to oldest
foreach my $iDbIdBackup (sort {$b <=> $a} keys %{$hDbListBackup})
{
if ($$hDbListBackup{$iDbIdBackup}{&INFO_SYSTEM_ID} == $ullDbSysIdArchive &&
$$hDbListBackup{$iDbIdBackup}{&INFO_DB_VERSION} eq $strDbVersionArchive)
{
$iDbHistoryId = $iDbIdBackup;
last;
}
}
}
# If the database is not found in the backup.info history list
if (!defined($iDbHistoryId))
{
# Check to see that the current DB sections match for the archive and backup info files
if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
($self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION)))) ||
!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef,
($self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID)))))
{
# This should never happen unless the backup.info file is corrupt
confess &log(ASSERT, "the archive and backup database sections do not match", ERROR_FILE_INVALID);
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iDbHistoryId', value => $iDbHistoryId}
);
}
####################################################################################################################################
# listByArchiveId
#
# Filters a list of backups by the archiveId passed.
####################################################################################################################################
sub listByArchiveId
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strArchiveId,
$strPathBackupArchive,
$stryBackup,
$strOrder,
) =
logDebugParam
(
__PACKAGE__ . '->listByArchiveId', \@_,
{name => 'strArchiveId'},
{name => 'strPathBackupArchive'},
{name => 'stryBackup'},
{name => 'strOrder', default => 'forward'}
);
# List of backups associated with the db-id provided
my @stryArchiveBackup;
my $iDbHistoryId = $self->backupArchiveDbHistoryId($strArchiveId, $strPathBackupArchive);
# If history found, then build list of backups associated with the archive id passed, else return empty array
if (defined($iDbHistoryId))
{
# Iterate through the backups and filter
foreach my $strBackup (@$stryBackup)
{
# From the backup.info current backup section, get the db-id for the backup and if it is the same, add to the list
if ($self->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_HISTORY_ID, $iDbHistoryId))
{
if ($strOrder eq 'reverse')
{
unshift(@stryArchiveBackup, $strBackup)
}
else
{
push(@stryArchiveBackup, $strBackup)
}
}
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'stryArchiveBackup', value => \@stryArchiveBackup}
);
}
####################################################################################################################################
# last
#
# Find the last backup depending on the type.
####################################################################################################################################
sub last
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strType
) =
logDebugParam
(
__PACKAGE__ . '->last', \@_,
{name => 'strType'}
);
my $strFilter = backupRegExpGet(true, $strType ne CFGOPTVAL_BACKUP_TYPE_FULL, $strType eq CFGOPTVAL_BACKUP_TYPE_INCR);
my $strBackup = ($self->list($strFilter, 'reverse'))[0];
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strBackup', value => $strBackup}
);
}
####################################################################################################################################
# delete
#
# Delete a backup from the info file.
####################################################################################################################################
sub delete
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strBackupLabel
) =
logDebugParam
(
__PACKAGE__ . '->delete', \@_,
{name => 'strBackupLabel'}
);
$self->remove(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel);
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# create
#
# Create the info file. WARNING - this file should only be called from stanza-create or test modules.
####################################################################################################################################
sub create
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$ullDbSysId,
$iControlVersion,
$iCatalogVersion,
$bSave,
) =
logDebugParam
(
__PACKAGE__ . '->create', \@_,
{name => 'strDbVersion'},
{name => 'ullDbSysId'},
{name => 'iControlVersion'},
{name => 'iCatalogVersion'},
{name => 'bSave', default => true},
);
# Fill db section and db history section
$self->dbSectionSet($strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId, $self->dbHistoryIdGet(false));
if ($bSave)
{
$self->save();
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# dbHistoryIdGet
#
# Get the db history ID
####################################################################################################################################
sub dbHistoryIdGet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$bFileRequired,
) = logDebugParam
(
__PACKAGE__ . '->dbHistoryIdGet', \@_,
{name => 'bFileRequired', default => true},
);
# Confirm the info file exists if it is required
if ($bFileRequired)
{
$self->confirmExists();
}
# If the DB section does not exist, initialize the history to one, else return the latest ID
my $iDbHistoryId = (!$self->test(INFO_BACKUP_SECTION_DB))
? 1 : $self->numericGet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iDbHistoryId', value => $iDbHistoryId}
);
}
####################################################################################################################################
# dbHistoryList
#
# Get the data from the db history section.
####################################################################################################################################
sub dbHistoryList
{
my $self = shift;
my
(
$strOperation,
) = logDebugParam
(
__PACKAGE__ . '->dbHistoryList',
);
my %hDbHash;
foreach my $iHistoryId ($self->keys(INFO_BACKUP_SECTION_DB_HISTORY))
{
$hDbHash{$iHistoryId}{&INFO_DB_VERSION} =
$self->get(INFO_BACKUP_SECTION_DB_HISTORY, $iHistoryId, INFO_BACKUP_KEY_DB_VERSION);
$hDbHash{$iHistoryId}{&INFO_SYSTEM_ID} =
$self->get(INFO_BACKUP_SECTION_DB_HISTORY, $iHistoryId, INFO_BACKUP_KEY_SYSTEM_ID);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hDbHash', value => \%hDbHash}
);
}
####################################################################################################################################
# dbSectionSet
#
# Set the db and db:history sections.
####################################################################################################################################
sub dbSectionSet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$iControlVersion,
$iCatalogVersion,
$ullDbSysId,
$iDbHistoryId,
) =
logDebugParam
(
__PACKAGE__ . '->dbSectionSet', \@_,
{name => 'strDbVersion', trace => true},
{name => 'iControlVersion', trace => true},
{name => 'iCatalogVersion', trace => true},
{name => 'ullDbSysId', trace => true},
{name => 'iDbHistoryId', trace => true},
);
# Fill db section
$self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG, undef, $iCatalogVersion);
$self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL, undef, $iControlVersion);
$self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID, undef, $ullDbSysId);
# Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one
$self->set(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, $strDbVersion . '');
$self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef, $iDbHistoryId);
# Fill db history
$self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_CATALOG, $iCatalogVersion);
$self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_CONTROL, $iControlVersion);
$self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_SYSTEM_ID, $ullDbSysId);
$self->set(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_DB_VERSION, $strDbVersion . '');
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# confirmDb
#
# Ensure that the backup is associated with the database passed.
# NOTE: The backup must exist in the backup:current section.
####################################################################################################################################
sub confirmDb
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strBackup,
$strDbVersion,
$ullDbSysId,
) =
logDebugParam
(
__PACKAGE__ . '->confirmDb', \@_,
{name => 'strBackup', trace => true},
{name => 'strDbVersion', trace => true},
{name => 'ullDbSysId', trace => true},
);
my $bConfirmDb = undef;
# Get the db-id associated with the backup
my $iDbHistoryId = $self->get(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_HISTORY_ID);
# Get the version and system-id for all known databases
my $hDbList = $self->dbHistoryList();
# If the db-id for the backup exists in the list
if (exists $hDbList->{$iDbHistoryId})
{
# If the version and system-id match then database is confirmed for the backup
if (($hDbList->{$iDbHistoryId}{&INFO_DB_VERSION} eq $strDbVersion) &&
($hDbList->{$iDbHistoryId}{&INFO_SYSTEM_ID} eq $ullDbSysId))
{
$bConfirmDb = true;
}
else
{
$bConfirmDb = false;
}
}
# If not, the backup.info file must be corrupt
else
{
confess &log(ERROR, "backup info file is missing database history information for an existing backup", ERROR_FILE_INVALID);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bConfirmDb', value => $bConfirmDb}
);
}
####################################################################################################################################
# confirmExists
#
# Ensure that the backup.info file and the db section exist.
####################################################################################################################################
sub confirmExists
{
my $self = shift;
# Confirm the file exists and the DB section is filled out
if (!$self->test(INFO_BACKUP_SECTION_DB) || !$self->{bExists})
{
confess &log(ERROR, $self->{strBackupClusterPath} . "/" . $strBackupInfoMissingMsg, ERROR_FILE_MISSING);
}
}
1;

639
test/lib/pgBackRestTest/Env/ExpireEnvTest.pm
View File

@ -1,639 +0,0 @@
####################################################################################################################################
# ExpireCommonTest.pm - Common code for expire tests
####################################################################################################################################
package pgBackRestTest::Env::ExpireEnvTest;
use parent 'pgBackRestTest::Env::HostEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Fcntl qw(O_RDONLY);
use File::Basename qw(basename);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Env::ArchiveInfo;
use pgBackRestTest::Env::BackupInfo;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{oHostBackup},
$self->{strBackRestExe},
$self->{oStorageRepo},
$self->{strPgPath},
$self->{oRunTest},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oHostBackup', required => false, trace => true},
{name => 'strBackRestExe', trace => true},
{name => 'oStorageRepo', trace => true},
{name => 'strPgPath', trace => true},
{name => 'oRunTest', required => false, trace => true},
);
$self->{strVm} = $self->{oRunTest}->vm();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# get into from pg_control
####################################################################################################################################
my $oPgControlVersionHash =
{
# iControlVersion => {iCatalogVersion => strDbVersion}
942 =>
{
201409291 => PG_VERSION_94,
201510051 => PG_VERSION_95,
},
960 =>
{
201608131 => PG_VERSION_96,
},
1002 =>
{
201707211 => PG_VERSION_10,
},
1100 =>
{
201809051 => PG_VERSION_11,
},
1201 =>
{
201909212 => PG_VERSION_12,
},
1300 =>
{
202007201 => PG_VERSION_13,
},
};
sub info
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbPath
) =
logDebugParam
(
__PACKAGE__ . '->info', \@_,
{name => 'strDbPath', default => $self->{strPgPath}}
);
# Open the control file and read system id and versions
#-----------------------------------------------------------------------------------------------------------------------
my $strControlFile = "${strDbPath}/" . DB_FILE_PGCONTROL;
my $hFile;
my $tBlock;
sysopen($hFile, $strControlFile, O_RDONLY)
or confess &log(ERROR, "unable to open ${strControlFile}", ERROR_FILE_OPEN);
# Read system identifier
sysread($hFile, $tBlock, 8) == 8
or confess &log(ERROR, "unable to read database system identifier");
$self->{info}{$strDbPath}{ullDbSysId} = unpack('Q', $tBlock);
# Read control version
sysread($hFile, $tBlock, 4) == 4
or confess &log(ERROR, "unable to read control version");
$self->{info}{$strDbPath}{iDbControlVersion} = unpack('L', $tBlock);
# Read catalog version
sysread($hFile, $tBlock, 4) == 4
or confess &log(ERROR, "unable to read catalog version");
$self->{info}{$strDbPath}{iDbCatalogVersion} = unpack('L', $tBlock);
# Close the control file
close($hFile);
# Get PostgreSQL version
$self->{info}{$strDbPath}{strDbVersion} =
$oPgControlVersionHash->{$self->{info}{$strDbPath}{iDbControlVersion}}
{$self->{info}{$strDbPath}{iDbCatalogVersion}};
if (!defined($self->{info}{$strDbPath}{strDbVersion}))
{
confess &log(
ERROR,
'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} .
' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" .
'HINT: is this version of PostgreSQL supported?');
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strDbVersion', value => $self->{info}{$strDbPath}{strDbVersion}},
{name => 'iDbControlVersion', value => $self->{info}{$strDbPath}{iDbControlVersion}},
{name => 'iDbCatalogVersion', value => $self->{info}{$strDbPath}{iDbCatalogVersion}},
{name => 'ullDbSysId', value => $self->{info}{$strDbPath}{ullDbSysId}}
);
}
####################################################################################################################################
# stanzaSet - set the local stanza object
####################################################################################################################################
sub stanzaSet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
$strDbVersion,
$bStanzaUpgrade,
) =
logDebugParam
(
__PACKAGE__ . '->stanzaSet', \@_,
{name => 'strStanza'},
{name => 'strDbVersion'},
{name => 'bStanzaUpgrade'},
);
# Assign variables
my $oStanza = {};
my $oArchiveInfo = {};
my $oBackupInfo = {};
my $iArchiveDbId = 1;
my $iBackupDbId = 1;
# If we're not upgrading, then create the info files
if (!$bStanzaUpgrade)
{
$oArchiveInfo =
new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath(), false,
{bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef});
$oBackupInfo =
new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath(), false,
{bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef});
}
# Else get the info data from disk
else
{
$oArchiveInfo =
new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath(),
{strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef});
$oBackupInfo =
new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath(),
{strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef});
}
# Get the database info for the stanza
(my $strVersion, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, $$oStanza{ullDbSysId}) = $self->info();
$$oStanza{strDbVersion} = $strDbVersion;
if ($bStanzaUpgrade)
{
$iArchiveDbId = $oArchiveInfo->dbHistoryIdGet() + 1;
$iBackupDbId = $oBackupInfo->dbHistoryIdGet() + 1;
}
$oArchiveInfo->dbSectionSet($$oStanza{strDbVersion}, $$oStanza{ullDbSysId}, $iArchiveDbId);
$oArchiveInfo->save();
$oBackupInfo->dbSectionSet($$oStanza{strDbVersion}, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion},
$$oStanza{ullDbSysId}, $iBackupDbId);
$oBackupInfo->save();
# Get the archive and directory paths for the stanza
$$oStanza{strArchiveClusterPath} = $self->{oHostBackup}->repoArchivePath($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oHostBackup}->repoBackupPath();
$self->{oStanzaHash}{$strStanza} = $oStanza;
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# stanzaCreate
####################################################################################################################################
sub stanzaCreate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
$strDbVersion,
) =
logDebugParam
(
__PACKAGE__ . '->stanzaCreate', \@_,
{name => 'strStanza'},
{name => 'strDbVersion'},
);
my $strDbVersionTemp = $strDbVersion;
$strDbVersionTemp =~ s/\.//;
# Create the test path for pg_control
storageTest()->pathCreate(($self->{strPgPath} . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true});
# Generate pg_control for stanza-create
$self->controlGenerate($self->{strPgPath}, $strDbVersion);
executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
# Create the stanza and set the local stanza object
$self->stanzaSet($strStanza, $strDbVersion, false);
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# stanzaUpgrade
####################################################################################################################################
sub stanzaUpgrade
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
$strDbVersion,
) =
logDebugParam
(
__PACKAGE__ . '->stanzaUpgrade', \@_,
{name => 'strStanza'},
{name => 'strDbVersion'},
);
my $strDbVersionTemp = $strDbVersion;
$strDbVersionTemp =~ s/\.//;
# Remove pg_control
storageTest()->remove($self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
# Copy pg_control for stanza-upgrade
$self->controlGenerate($self->{strPgPath}, $strDbVersion);
executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
$self->stanzaSet($strStanza, $strDbVersion, true);
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# backupCreate
####################################################################################################################################
sub backupCreate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
$strType,
$lTimestamp,
$iArchiveBackupTotal,
$iArchiveBetweenTotal
) =
logDebugParam
(
__PACKAGE__ . '->backupCreate', \@_,
{name => 'strStanza'},
{name => 'strType'},
{name => 'lTimestamp'},
{name => 'iArchiveBackupTotal', default => 3},
{name => 'iArchiveBetweenTotal', default => 3}
);
my $oStanza = $self->{oStanzaHash}{$strStanza};
my ($strArchiveStart, $strArchiveStop);
if ($iArchiveBackupTotal != -1)
{
($strArchiveStart, $strArchiveStop) = $self->archiveCreate($strStanza, $iArchiveBackupTotal);
}
# Create the manifest
my $oLastManifest = $strType ne CFGOPTVAL_BACKUP_TYPE_FULL ? $$oStanza{oManifest} : undef;
my $strBackupLabel =
backupLabelFormat($strType,
defined($oLastManifest) ? $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) : undef,
$lTimestamp);
my $strBackupClusterSetPath = "$$oStanza{strBackupClusterPath}/${strBackupLabel}";
&log(INFO, "create backup ${strBackupLabel}");
# Get passphrase (returns undefined if repo not encrypted) to access the manifest
my $strCipherPassManifest =
(new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath()))->cipherPassSub();
my $strCipherPassBackupSet;
# If repo is encrypted then get passphrase for accessing the backup files from the last manifest if it exists provide one
if (defined($strCipherPassManifest))
{
$strCipherPassBackupSet = (defined($oLastManifest)) ? $oLastManifest->cipherPassSub() :
ENCRYPTION_KEY_BACKUPSET;
}
my $strManifestFile = "$$oStanza{strBackupClusterPath}/${strBackupLabel}/" . FILE_MANIFEST;
my $oManifest = new pgBackRestTest::Env::Manifest($strManifestFile, {bLoad => false, strDbVersion => PG_VERSION_94,
iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94),
strCipherPass => $strCipherPassManifest, strCipherPassSub => $strCipherPassBackupSet});
# Store information about the backup into the backup section
$oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $strBackupLabel);
$oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK, undef, true);
$oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY, undef, false);
$oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, false);
$oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, $strArchiveStart);
$oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, $strArchiveStop);
$oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, 'backup-bundle', undef, true);
$oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE, undef, true);
$oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, true);
$oManifest->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, REPOSITORY_FORMAT);
$oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK, undef, false);
$oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, true);
$oManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, $lTimestamp);
$oManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, $lTimestamp);
$oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, $strType);
$oManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, PROJECT_VERSION);
if ($strType ne CFGOPTVAL_BACKUP_TYPE_FULL)
{
if (!defined($oLastManifest))
{
confess &log(ERROR, "oLastManifest must be defined when strType = ${strType}");
}
# Set backup-prior
if ($strType eq CFGOPTVAL_BACKUP_TYPE_INCR)
{
# If this is an incremental backup, then it is always based on the prior backup so use the label from the last backup
$oManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef,
$oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL));
}
else
{
# If it is a differential then backup-prior must be set to the newest full backup so get the full backup label from
# the prior label
$oManifest->set(
MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef,
substr($oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL), 0, 16));
}
}
$oManifest->save();
$$oStanza{oManifest} = $oManifest;
# Add the backup to info
my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($$oStanza{strBackupClusterPath}, false);
$oBackupInfo->check($$oStanza{strDbVersion}, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, $$oStanza{ullDbSysId});
$oBackupInfo->add($oManifest);
# Create the backup description string
if (defined($$oStanza{strBackupDescription}))
{
$$oStanza{strBackupDescription} .= "\n";
}
$$oStanza{strBackupDescription} .=
"* ${strType} backup: label = ${strBackupLabel}" .
(defined($oLastManifest) ? ', prior = ' . $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) : '') .
(defined($strArchiveStart) ? ", start = ${strArchiveStart}, stop = ${strArchiveStop}" : ', not online');
if ($iArchiveBetweenTotal != -1)
{
$self->archiveCreate($strStanza, $iArchiveBetweenTotal);
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# archiveNext
####################################################################################################################################
sub archiveNext
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strArchive,
$bSkipFF
) =
logDebugParam
(
__PACKAGE__ . '->archiveNext', \@_,
{name => 'strArchive', trace => true},
{name => 'bSkipFF', trace => true}
);
# Break archive log into components
my $lTimeline = hex(substr($strArchive, 0, 8));
my $lMajor = hex(substr($strArchive, 8, 8));
my $lMinor = hex(substr($strArchive, 16, 8));
# Increment the minor component (and major when needed)
$lMinor += 1;
if ($bSkipFF && $lMinor == 255 || !$bSkipFF && $lMinor == 256)
{
$lMajor += 1;
$lMinor = 0;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveNext', value => uc(sprintf("%08x%08x%08x", $lTimeline, $lMajor, $lMinor)), trace => true}
);
}
####################################################################################################################################
# archiveCreate
####################################################################################################################################
sub archiveCreate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
$iArchiveTotal
) =
logDebugParam
(
__PACKAGE__ . '->archiveCreate', \@_,
{name => 'strStanza'},
{name => 'iArchiveTotal'}
);
my $oStanza = $self->{oStanzaHash}{$strStanza};
my $iArchiveIdx = 0;
my $strArchive = defined($$oStanza{strArchiveLast}) ? $self->archiveNext($$oStanza{strArchiveLast}, false) :
'000000010000000000000000';
# Get passphrase (returns undefined if repo not encrypted) to access the archive files
my $strCipherPass =
(new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath()))->cipherPassSub();
push(my @stryArchive, $strArchive);
do
{
my $strPath = "$$oStanza{strArchiveClusterPath}/" . substr($strArchive, 0, 16);
my $strFile = "${strPath}/${strArchive}-0000000000000000000000000000000000000000" . ($iArchiveIdx % 2 == 0 ? '.gz' : '');
storageRepo()->put($strFile, 'ARCHIVE', {strCipherPass => $strCipherPass});
$iArchiveIdx++;
if ($iArchiveIdx < $iArchiveTotal)
{
$strArchive = $self->archiveNext($strArchive, false);
}
}
while ($iArchiveIdx < $iArchiveTotal);
push(@stryArchive, $strArchive);
$$oStanza{strArchiveLast} = $strArchive;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'stryArchive', value => \@stryArchive}
);
}
####################################################################################################################################
# process
####################################################################################################################################
sub process
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
$iExpireFull,
$iExpireDiff,
$strExpireArchiveType,
$iExpireArchive,
$strDescription
) =
logDebugParam
(
__PACKAGE__ . '->process', \@_,
{name => 'strStanza'},
{name => 'iExpireFull', required => false},
{name => 'iExpireDiff', required => false},
{name => 'strExpireArchiveType'},
{name => 'iExpireArchive', required => false},
{name => 'strDescription'}
);
my $oStanza = $self->{oStanzaHash}{$strStanza};
undef($$oStanza{strBackupDescription});
my $strCommand =
$self->{strBackRestExe} . ' --config="' . $self->{oHostBackup}->backrestConfig() . '"' . ' --stanza=' . $strStanza .
' --log-level-console=' . lc(DETAIL);
if (defined($iExpireFull))
{
$strCommand .= ' --repo1-retention-full=' . $iExpireFull;
}
if (defined($iExpireDiff))
{
$strCommand .= ' --repo1-retention-diff=' . $iExpireDiff;
}
if (defined($strExpireArchiveType))
{
if (defined($iExpireArchive))
{
$strCommand .= ' --repo1-retention-archive-type=' . $strExpireArchiveType .
' --repo1-retention-archive=' . $iExpireArchive;
}
else
{
$strCommand .= ' --repo1-retention-archive-type=' . $strExpireArchiveType;
}
}
$strCommand .= ' expire';
$self->{oHostBackup}->executeSimple($strCommand, {strComment => $strDescription});
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
1;

82
test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm
View File

@ -1,82 +0,0 @@
####################################################################################################################################
# Azure Test Host
####################################################################################################################################
package pgBackRestTest::Env::Host::HostAzureTest;
use parent 'pgBackRestTest::Common::HostTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# Azure defaults
####################################################################################################################################
use constant HOST_AZURE_ACCOUNT => 'azaccount';
push @EXPORT, qw(HOST_AZURE_ACCOUNT);
use constant HOST_AZURE_KEY => 'YXpLZXk=';
push @EXPORT, qw(HOST_AZURE_KEY);
use constant HOST_AZURE_CONTAINER => 'azcontainer';
push @EXPORT, qw(HOST_AZURE_CONTAINER);
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
);
# Create the host
my $strProjectPath = dirname(dirname(abs_path($0)));
my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert";
my $self = $class->SUPER::new(
HOST_AZURE, 'test-' . testRunGet()->vmId() . '-' . HOST_AZURE, 'mcr.microsoft.com/azure-storage/azurite', 'root',
["${strFakeCertPath}/s3-server.crt:/root/public.crt:ro", "${strFakeCertPath}/s3-server.key:/root/private.key:ro"],
'-e AZURITE_ACCOUNTS="' . HOST_AZURE_ACCOUNT . ':' . HOST_AZURE_KEY . '"',
'azurite-blob --blobPort 443 --blobHost 0.0.0.0 --cert=/root/public.crt --key=/root/private.key -d debug.log"
" --disableProductStyleUrl',
false);
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
1;

2321
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
View File

File diff suppressed because it is too large Load Diff

139
test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm
View File

@ -1,139 +0,0 @@
####################################################################################################################################
# HostBackupTest.pm - Backup host
####################################################################################################################################
package pgBackRestTest::Env::Host::HostBaseTest;
use parent 'pgBackRestTest::Common::HostTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::JobTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::VmTest;
####################################################################################################################################
# Host constants
####################################################################################################################################
use constant HOST_BASE => 'base';
push @EXPORT, qw(HOST_BASE);
use constant HOST_DB_PRIMARY => 'db-primary';
push @EXPORT, qw(HOST_DB_PRIMARY);
use constant HOST_DB_STANDBY => 'db-standby';
push @EXPORT, qw(HOST_DB_STANDBY);
use constant HOST_BACKUP => 'backup';
push @EXPORT, qw(HOST_BACKUP);
use constant HOST_GCS => 'gcs';
push @EXPORT, qw(HOST_GCS);
use constant HOST_AZURE => 'azure';
push @EXPORT, qw(HOST_AZURE);
use constant HOST_S3 => 's3-server';
push @EXPORT, qw(HOST_S3);
use constant HOST_SFTP => 'sftp-srvr';
push @EXPORT, qw(HOST_SFTP);
####################################################################################################################################
# CA/cert/key constants
####################################################################################################################################
use constant HOST_CERT_PATH => '/test/certificate/';
use constant HOST_CLIENT_CERT => HOST_CERT_PATH . 'pgbackrest-test-client.crt';
push @EXPORT, qw(HOST_CLIENT_CERT);
use constant HOST_CLIENT_KEY => HOST_CERT_PATH . 'pgbackrest-test-client.key';
push @EXPORT, qw(HOST_CLIENT_KEY);
use constant HOST_SERVER_CA => HOST_CERT_PATH . 'pgbackrest-test-ca.crt';
push @EXPORT, qw(HOST_SERVER_CA);
use constant HOST_SERVER_CERT => HOST_CERT_PATH . 'pgbackrest-test-server.crt';
use constant HOST_SERVER_KEY => HOST_CERT_PATH . 'pgbackrest-test-server.key';
####################################################################################################################################
# SFTP key constants
####################################################################################################################################
use constant SSH_KEY_PATH => '/test/certificate/ssh/';
use constant SSH_PRIVATE_KEY => SSH_KEY_PATH . 'id_rsa';
push @EXPORT, qw(SSH_PRIVATE_KEY);
use constant SSH_PUBLIC_KEY => SSH_KEY_PATH . 'id_rsa.pub';
push @EXPORT, qw(SSH_PUBLIC_KEY);
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strName,
$oParam,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strName', default => HOST_BASE, trace => true},
{name => 'oParam', required => false, trace => true},
);
my $strTestPath = testRunGet()->testPath() . ($strName eq HOST_BASE ? '' : "/${strName}");
storageTest()->pathCreate($strTestPath, {strMode => '0770'});
# Make sure keys have the correct permissions
if (chmod(0600, testRunGet()->basePath() . HOST_SERVER_KEY, testRunGet()->basePath() . HOST_CLIENT_KEY) != 2)
{
confess "unable to set mode on keys";
}
# Create the host
my $strProjectPath = dirname(dirname(abs_path($0)));
my $strBinPath = dirname(dirname($strTestPath)) . '/bin/' . testRunGet()->vm() . '/' . PROJECT_EXE;
my $strContainer = 'test-' . testRunGet()->vmId() . "-$strName";
my $self = $class->SUPER::new(
$strName, $strContainer, $$oParam{strImage}, $$oParam{strUser},
["${strProjectPath}:${strProjectPath}", "${strTestPath}:${strTestPath}", "${strBinPath}:${strBinPath}:ro"], undef,
$oParam->{bTls} ?
'server --log-level-console=debug --tls-server-ca-file=' . testRunGet()->basePath() . HOST_SERVER_CA .
' --tls-server-cert-file=' . testRunGet()->basePath() . HOST_SERVER_CERT . ' --tls-server-key-file=' .
testRunGet()->basePath() . HOST_SERVER_KEY . ' --tls-server-auth=pgbackrest-client=* --tls-server-address=0.0.0.0' :
undef,
undef, $oParam->{bTls} ? testRunGet()->backrestExe() : undef);
bless $self, $class;
# Set test path
$self->{strTestPath} = $strTestPath;
# Set permissions on the test path
$self->executeSimple('chown -R ' . $self->userGet() . ':'. TEST_GROUP . ' ' . $self->testPath(), undef, 'root');
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub testPath {return shift->{strTestPath}}
1;

217
test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm
View File

@ -1,217 +0,0 @@
####################################################################################################################################
# HostDbTest.pm - Database host
####################################################################################################################################
package pgBackRestTest::Env::Host::HostDbCommonTest;
use parent 'pgBackRestTest::Env::Host::HostBackupTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::Common::String;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# Test WAL size
####################################################################################################################################
use constant PG_WAL_SIZE_TEST => 16777216;
####################################################################################################################################
# Host defaults
####################################################################################################################################
use constant HOST_PATH_SPOOL => 'spool';
use constant HOST_PATH_DB => 'db';
use constant HOST_PATH_DB_BASE => 'base';
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oParam,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oParam', required => false, trace => true},
);
# Get host group
my $oHostGroup = hostGroupGet();
# Is standby?
my $bStandby = defined($$oParam{bStandby}) && $$oParam{bStandby} ? true : false;
my $self = $class->SUPER::new(
{
strName => $bStandby ? HOST_DB_STANDBY : HOST_DB_PRIMARY,
strImage => $$oParam{strImage},
bTls => $oParam->{bTls},
strBackupDestination => $$oParam{strBackupDestination},
bSynthetic => $$oParam{bSynthetic},
bRepoLocal => $oParam->{bRepoLocal},
bRepoEncrypt => $oParam->{bRepoEncrypt},
});
bless $self, $class;
# Set parameters
$self->{bStandby} = $bStandby;
$self->{strDbPath} = $self->testPath() . '/' . HOST_PATH_DB;
$self->{strDbBasePath} = $self->dbPath() . '/' . HOST_PATH_DB_BASE;
$self->{strTablespacePath} = $self->dbPath() . '/tablespace';
storageTest()->pathCreate($self->dbBasePath(), {strMode => '0700', bCreateParent => true});
$self->{strSpoolPath} = $self->testPath() . '/' . HOST_PATH_SPOOL;
storageTest()->pathCreate($self->spoolPath());
# Initialize linkRemap Hashes
$self->{hLinkRemap} = {};
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# archivePush
####################################################################################################################################
sub archivePush
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strWalPath,
$strArchiveTestFile,
$iArchiveNo,
$iExpectedError,
$bAsync,
$strOptionalParam,
) =
logDebugParam
(
__PACKAGE__ . '->archivePush', \@_,
{name => 'strWalPath'},
{name => 'strArchiveTestFile', required => false},
{name => 'iArchiveNo', required => false},
{name => 'iExpectedError', required => false},
{name => 'bAsync', default => true},
{name => 'strOptionalParam', required => false},
);
my $strSourceFile;
if (defined($strArchiveTestFile))
{
$strSourceFile = "${strWalPath}/" . uc(sprintf('0000000100000001%08x', $iArchiveNo));
storageTest()->copy($strArchiveTestFile, storageTest()->openWrite($strSourceFile, {bPathCreate => true}));
storageTest()->pathCreate("${strWalPath}/archive_status/", {bIgnoreExists => true, bCreateParent => true});
storageTest()->put("${strWalPath}/archive_status/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)) . '.ready');
}
$self->executeSimple(
$self->backrestExe() .
' --config=' . $self->backrestConfig() .
' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE_TEST) .
' --stanza=' . $self->stanza() .
($bAsync ? '' : ' --no-archive-async') .
" archive-push" . (defined($strSourceFile) ? " ${strSourceFile}" : '') .
(defined($strOptionalParam) ? " ${strOptionalParam}" : ''),
{iExpectedExitStatus => $iExpectedError, bLogOutput => $self->synthetic()});
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# linkRemap
####################################################################################################################################
sub linkRemap
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strTarget,
$strDestination
) =
logDebugParam
(
__PACKAGE__ . '->linkRemap', \@_,
{name => 'strTarget'},
{name => 'strDestination'},
);
${$self->{hLinkRemap}}{$strTarget} = $strDestination;
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub dbPath {return shift->{strDbPath};}
sub dbBasePath
{
my $self = shift;
my $iIndex = shift;
return $self->{strDbBasePath} . (defined($iIndex) ? "-${iIndex}" : '');
}
sub spoolPath {return shift->{strSpoolPath}}
sub standby {return shift->{bStandby}}
sub tablespacePath
{
my $self = shift;
my $iTablespace = shift;
my $iIndex = shift;
return
$self->{strTablespacePath} .
(defined($iTablespace) ? "/ts${iTablespace}" .
(defined($iIndex) ? "-${iIndex}" : '') : '');
}
1;

717
test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm
View File

@ -1,717 +0,0 @@
####################################################################################################################################
# HostDbTest.pm - Database host
####################################################################################################################################
package pgBackRestTest::Env::Host::HostDbSyntheticTest;
use parent 'pgBackRestTest::Env::Host::HostDbCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use Fcntl ':mode';
use File::Basename qw(basename dirname);
use File::stat;
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::Common::String;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Host::HostDbCommonTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oParam,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oParam', required => false, trace => true},
);
my $self = $class->SUPER::new(
{
strImage => containerRepo() . ':' . testRunGet()->vm() . "-test",
bTls => $oParam->{bTls},
strBackupDestination => $$oParam{strBackupDestination},
bSynthetic => true,
bStandby => $$oParam{bStandby},
bRepoLocal => $oParam->{bRepoLocal},
bRepoEncrypt => $oParam->{bRepoEncrypt},
});
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# dbFileCreate
#
# Create a file specifying content, mode, and time.
####################################################################################################################################
sub dbFileCreate
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strFile = shift;
my $strContent = shift;
my $lTime = shift;
my $strMode = shift;
# Check that strTarget is a valid
my $strPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH};
if (!defined($strPath))
{
confess &log(ERROR, "${strTarget} not a valid target: \n" . Dumper(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}));
}
# Get tablespace path if this is a tablespace
my $strPgPath;
if (index($strTarget, DB_PATH_PGTBLSPC . '/') == 0)
{
my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG};
$strPgPath = 'PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}";
}
# Create actual file location
my $strPathFile = $strPath .
(defined($strPgPath) ? "/${strPgPath}" : '') . "/${strFile}";
if (index($strPathFile, '/') != 0)
{
$strPathFile = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . '/' .
(defined(dirname($strPathFile)) ? dirname($strPathFile) : '') . "/${strPathFile}";
}
# Create the file
testFileCreate($strPathFile, $strContent, $lTime, $strMode);
# Return path to created file
return $strPathFile;
}
####################################################################################################################################
# dbFileRemove
####################################################################################################################################
sub dbFileRemove
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strFile = shift;
my $bIgnoreMissing = shift;
# Get actual path location
my $strDbFile = $self->manifestDbPathGet($oManifestRef, $strTarget, $strFile);
# Remove the file
if (!(defined($bIgnoreMissing) && $bIgnoreMissing && !(-e $strDbFile)))
{
testFileRemove($strDbFile);
}
return $strDbFile;
}
####################################################################################################################################
# dbLinkCreate
#
# Create a file specifying content, mode, and time.
####################################################################################################################################
sub dbLinkCreate
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strFile = shift;
my $strDestination = shift;
# Create actual file location
my $strDbFile = $self->manifestDbPathGet($oManifestRef, $strTarget, $strFile);
# Create the file
testLinkCreate($strDbFile, $strDestination);
# Return path to created file
return $strDbFile;
}
####################################################################################################################################
# manifestDbPathGet
#
# Get the db path based on the target and file passed.
####################################################################################################################################
sub manifestDbPathGet
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strFile = shift;
# Determine the manifest key
my $strDbPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH};
# If target is a tablespace
if (defined(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID}))
{
my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG};
$strDbPath .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}";
}
$strDbPath .= defined($strFile) ? "/${strFile}" : '';
return $strDbPath;
}
####################################################################################################################################
# manifestFileCreate
#
# Create a file specifying content, mode, and time and add it to the manifest.
####################################################################################################################################
sub manifestFileCreate
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strFile = shift;
my $strContent = shift;
my $strChecksum = shift;
my $lTime = shift;
my $strMode = shift;
my $bPrimary = shift;
my $strChecksumPageError = shift;
# Determine the manifest key
my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strTarget, $strFile);
# Create the file
my $strPathFile = $self->dbFileCreate($oManifestRef, $strTarget, $strFile, $strContent, $lTime, $strMode);
# Stat the file
my $oStat = lstat($strPathFile);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} =
sprintf('%04o', S_IMODE($oStat->mode));
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime;
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size;
delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_REFERENCE});
my $bChecksumPage = defined($strChecksumPageError) ? false : (isChecksumPage($strManifestKey) ? true : undef);
if (defined($bChecksumPage))
{
$oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE} =
$bChecksumPage ? JSON::PP::true : JSON::PP::false;
if (!$bChecksumPage && $strChecksumPageError ne '0')
{
my @iyChecksumPageError = eval($strChecksumPageError);
$oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR} =
\@iyChecksumPageError;
}
else
{
delete($oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR});
}
}
if (defined($strChecksum))
{
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{checksum} = $strChecksum;
}
}
####################################################################################################################################
# manifestFileRemove
#
# Remove a file from disk and (optionally) the manifest.
####################################################################################################################################
sub manifestFileRemove
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strFile = shift;
# Determine the manifest key
my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strTarget, $strFile);
# Remove the file
$self->dbFileRemove($oManifestRef, $strTarget, $strFile, true);
# Remove from manifest
delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey});
}
####################################################################################################################################
# manifestKeyGet
#
# Get the manifest key based on the target and file/path/link passed.
####################################################################################################################################
sub manifestKeyGet
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strFile = shift;
# Determine the manifest key
my $strManifestKey = $strTarget;
# If target is a tablespace
if (defined(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID}))
{
my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG};
$strManifestKey .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}";
}
$strManifestKey .= (defined($strFile) ? "/$strFile" : '');
return $strManifestKey;
}
####################################################################################################################################
# manifestLinkCreate
#
# Create a link and add it to the manifest.
####################################################################################################################################
sub manifestLinkCreate
{
my $self = shift;
my $oManifestRef = shift;
my $strPath = shift;
my $strFile = shift;
my $strDestination = shift;
my $bPrimary = shift;
# Determine the manifest key
my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strFile);
# Load target
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH} = $strDestination;
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_LINK;
# Create the link
my $strDbFile = $self->dbLinkCreate($oManifestRef, $strPath, $strFile, $strDestination);
# Stat the link
my $oStat = lstat($strDbFile);
# Check for errors in stat
if (!defined($oStat))
{
confess 'unable to stat ${strDbFile}';
}
# Load file into manifest
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_DESTINATION} = $strDestination;
# Stat what the link is pointing to
my $strDestinationFile = $strDestination;
if (index($strDestinationFile, '/') != 0)
{
$strDestinationFile = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . '/' .
(defined(dirname($strPath)) ? dirname($strPath) : '') . "/${strDestination}";
}
$oStat = lstat($strDestinationFile);
my $strSection = MANIFEST_SECTION_TARGET_PATH;
if (S_ISREG($oStat->mode))
{
$strSection = MANIFEST_SECTION_TARGET_FILE;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime;
(${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM}) = storageTest()->hashSize($strDestinationFile);
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_FILE} =
basename(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH});
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH} =
dirname(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH});
}
# Allow a link to a link to be created to test that backrest errors out correctly
elsif (S_ISLNK($oStat->mode))
{
$strSection = MANIFEST_SECTION_TARGET_LINK;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_DESTINATION} = $strDestination;
}
elsif (!S_ISDIR($oStat->mode))
{
confess &log(ASSERT, "unrecognized file type for file $strDestinationFile");
}
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = sprintf('%04o', S_IMODE($oStat->mode));
}
####################################################################################################################################
# manifestLinkMap
#
# Remap links to new directories/files
####################################################################################################################################
sub manifestLinkMap
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strDestination = shift;
if ($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK)
{
confess "cannot map target ${strTarget} because it is not a link";
}
if (defined($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID}))
{
confess "tablespace ${strTarget} cannot be remapped with this function";
}
if (defined($strDestination))
{
confess "GENERAL LINK REMAP NOT IMPLEMENTED";
}
else
{
delete($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget});
delete($$oManifestRef{&MANIFEST_SECTION_TARGET_LINK}{$strTarget});
}
}
####################################################################################################################################
# manifestLinkRemove
#
# Create a link and add it to the manifest.
####################################################################################################################################
sub manifestLinkRemove
{
my $self = shift;
my $oManifestRef = shift;
my $strPath = shift;
my $strFile = shift;
# Delete the link
my $strDbFile = $self->dbFileRemove($oManifestRef, $strPath, $strFile);
# Determine the manifest key
my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strFile);
# Delete from manifest
delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey});
delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey});
delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey});
delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strManifestKey});
}
####################################################################################################################################
# manifestPathCreate
#
# Create a path specifying mode and add it to the manifest.
####################################################################################################################################
sub manifestPathCreate
{
my $self = shift;
my $oManifestRef = shift;
my $strPath = shift;
my $strSubPath = shift;
my $strMode = shift;
# Determine the manifest key
my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strSubPath);
# Create the db path
my $strDbPath = $self->dbPathCreate($oManifestRef, $strPath, $strSubPath, defined($strMode) ? $strMode : '0700');
# Stat the file
my $oStat = lstat($strDbPath);
# Check for errors in stat
if (!defined($oStat))
{
confess 'unable to stat ${strSubPath}';
}
# Load file into manifest
my $strSection = MANIFEST_SECTION_TARGET_PATH;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = sprintf('%04o', S_IMODE($oStat->mode));
}
####################################################################################################################################
# manifestReference
#
# Update all files that do not have a reference with the supplied reference.
####################################################################################################################################
sub manifestReference
{
my $self = shift;
my $oManifestRef = shift;
my $strReference = shift;
my $bClear = shift;
# Set prior backup
if (defined($strReference))
{
${$oManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} = $strReference;
}
else
{
delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR});
}
# Find all file sections
foreach my $strSectionFile (sort(keys(%$oManifestRef)))
{
# Skip non-file sections
if ($strSectionFile !~ /\:file$/)
{
next;
}
foreach my $strFile (sort(keys(%{${$oManifestRef}{$strSectionFile}})))
{
if (!defined($strReference))
{
delete(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE});
}
elsif (defined($bClear) && $bClear)
{
if (defined(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}) &&
${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE} ne $strReference)
{
delete(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE});
}
}
elsif (!defined(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}))
{
${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE} = $strReference;
}
}
}
}
####################################################################################################################################
# manifestTablespaceCreate
#
# Create a tablespace specifying mode and add it to the manifest.
####################################################################################################################################
sub manifestTablespaceCreate
{
my $self = shift;
my $oManifestRef = shift;
my $iOid = shift;
my $strMode = shift;
# Load linked path into manifest
my $strLinkPath = $self->tablespacePath($iOid);
my $strTarget = MANIFEST_TARGET_PGTBLSPC . "/${iOid}";
my $oStat = lstat($strLinkPath);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_MODE} =
sprintf('%04o', S_IMODE($oStat->mode));
# Create the tablespace path if it does not exist
my $strTablespacePath = $strLinkPath;
my $strPathTarget = $strTarget;
my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG};
my $strTablespaceId = 'PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}";
$strTablespacePath .= "/${strTablespaceId}";
$strPathTarget .= "/${strTablespaceId}";
if (!-e $strTablespacePath)
{
storageTest()->pathCreate($strTablespacePath, {strMode => defined($strMode) ? $strMode : '0700'});
}
# Load tablespace path into manifest
$oStat = lstat($strTablespacePath);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGTBLSPC} =
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA};
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_MODE} =
sprintf('%04o', S_IMODE($oStat->mode));
# Create the link in DB_PATH_PGTBLSPC
my $strLink = $self->dbBasePath() . '/' . DB_PATH_PGTBLSPC . "/${iOid}";
symlink($strLinkPath, $strLink)
or confess "unable to link ${strLink} to ${strLinkPath}";
# Load link into the manifest
$oStat = lstat($strLink);
my $strLinkTarget = MANIFEST_TARGET_PGDATA . "/${strTarget}";
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_DESTINATION} = $strLinkPath;
# Load tablespace target into the manifest
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH} = $strLinkPath;
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_LINK;
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID} = $iOid;
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME} = "ts${iOid}";
}
####################################################################################################################################
# manifestTablespaceDrop
#
# Drop a tablespace add remove it from the manifest.
####################################################################################################################################
sub manifestTablespaceDrop
{
my $self = shift;
my $oManifestRef = shift;
my $iOid = shift;
my $iIndex = shift;
# Remove tablespace path/file/link from manifest
my $strTarget = DB_PATH_PGTBLSPC . "/${iOid}";
# Remove manifest path, link, target
delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget});
delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{&MANIFEST_TARGET_PGDATA . "/${strTarget}"});
delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget});
# Remove nested manifest files and paths
foreach my $strSection (&MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_FILE)
{
foreach my $strFile (keys(%{${$oManifestRef}{$strSection}}))
{
if (index($strFile, "${strTarget}/") == 0)
{
delete($$oManifestRef{$strSection}{$strFile});
}
}
}
# Drop the link in DB_PATH_PGTBLSPC
testFileRemove($self->dbBasePath($iIndex) . "/${strTarget}");
}
####################################################################################################################################
# dbPathCreate
#
# Create a path specifying mode.
####################################################################################################################################
sub dbPathCreate
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strSubPath = shift;
my $strMode = shift;
# Create final file location
my $strFinalPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH};
# Get tablespace path if this is a tablespace
if (index($strTarget, DB_PATH_PGTBLSPC . '/') == 0)
{
my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG};
$strFinalPath .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}";
}
$strFinalPath .= (defined($strSubPath) ? "/${strSubPath}" : '');
# Create the path
if (!(-e $strFinalPath))
{
storageTest()->pathCreate($strFinalPath, {strMode => $strMode});
}
return $strFinalPath;
}
####################################################################################################################################
# dbPathMode
#
# Change the mode of a path.
####################################################################################################################################
sub dbPathMode
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strPath = shift;
my $strMode = shift;
# Get the db path
my $strDbPath = $self->manifestDbPathGet($oManifestRef, $strTarget, $strPath);
testPathMode($strDbPath, $strMode);
return $strDbPath;
}
####################################################################################################################################
# dbPathRemove
#
# Remove a path.
####################################################################################################################################
sub dbPathRemove
{
my $self = shift;
my $oManifestRef = shift;
my $strTarget = shift;
my $strPath = shift;
# Get the db path
my $strDbPath = $self->manifestDbPathGet($oManifestRef, $strTarget, $strPath);
# Create the path
testPathRemove($strDbPath);
return $strDbPath;
}
1;

528
test/lib/pgBackRestTest/Env/Host/HostDbTest.pm
View File

@ -1,528 +0,0 @@
####################################################################################################################################
# HostDbTest.pm - Database host
####################################################################################################################################
package pgBackRestTest::Env::Host::HostDbTest;
use parent 'pgBackRestTest::Env::Host::HostDbCommonTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use DBI;
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(basename);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::Common::String;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Host::HostDbCommonTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# Db defaults
####################################################################################################################################
use constant HOST_DB_DEFAULT => 'postgres';
use constant HOST_DB_TIMEOUT => 30;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oParam,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oParam', required => false, trace => true},
);
# Get db version
my $strDbVersion = testRunGet()->pgVersion();
my $self = $class->SUPER::new(
{
strImage => containerRepo() . ':' . testRunGet()->vm() . "-test",
bTls => $oParam->{bTls},
strBackupDestination => $$oParam{strBackupDestination},
bStandby => $$oParam{bStandby},
bRepoLocal => $oParam->{bRepoLocal},
bRepoEncrypt => $oParam->{bRepoEncrypt},
});
bless $self, $class;
# Set parameters
$self->{strPgSocketPath} = $self->dbPath();
$self->{iPgPort} = defined($$oParam{bStandby}) && $$oParam{bStandby} ? 6544 : 6543;
$self->{strPgLogPath} = $self->testPath();
$self->{strPgLogFile} = $self->pgLogPath() . '/postgresql.log';
# Get Db version
if (defined($strDbVersion))
{
my $strOutLog = $self->executeSimple($self->pgBinPath() . '/postgres --version');
my @stryVersionToken = split(/ /, $strOutLog);
@stryVersionToken = split(/\./, $stryVersionToken[2]);
my $strDbVersionActual =
trim($stryVersionToken[0]) .
(defined($stryVersionToken[1]) && trim($stryVersionToken[0]) < 10 ? '.' . trim($stryVersionToken[1]) : '');
# Warn if this is a devel/alpha/beta version
my $strVersionRegExp = '(devel|((alpha|beta|rc)[0-9]+))$';
if ($strDbVersionActual =~ /$strVersionRegExp/)
{
my $strDevVersion = $strDbVersionActual;
$strDbVersionActual =~ s/$strVersionRegExp//;
$strDevVersion = substr($strDevVersion, length($strDbVersionActual));
if (!defined($$oParam{bStandby}) || !$$oParam{bStandby})
{
&log(WARN, 'Testing against ' . trim($strOutLog) . " ${strDevVersion}");
}
}
elsif (!defined($$oParam{bStandby}) || !$$oParam{bStandby})
{
&log(INFO, 'Testing against ' . trim($strOutLog));
}
# Don't run unit tests for unsupported versions
my @stryVersionSupport = versionSupport();
if ($strDbVersionActual < $stryVersionSupport[0])
{
confess &log(ERROR, "only PostgreSQL version $stryVersionSupport[0] and up are supported");
}
if ($strDbVersion ne $strDbVersionActual)
{
confess &log(ERROR, "actual database version ${strDbVersionActual} does not match expected version ${strDbVersion}");
}
}
# Create wal directory
storageTest()->pathCreate($self->dbPath() . '/pg_' . $self->walId(), {strMode => '0700'});
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# sqlConnect
####################################################################################################################################
sub sqlConnect
{
my $self = shift;
my $hParam = shift;
# Set defaults
my $iTimeout = defined($$hParam{iTimeout}) ? $$hParam{iTimeout} : HOST_DB_TIMEOUT;
my $strDb = defined($$hParam{strDb}) ? $$hParam{strDb} : HOST_DB_DEFAULT;
# If not connected
if (!defined($self->{db}{$strDb}{hDb}))
{
# Retry until connection is successful
my $oWait = waitInit($iTimeout);
do
{
# Connect to the db (whether it is local or remote)
$self->{db}{$strDb}{hDb} =
DBI->connect(
"dbi:Pg:dbname=${strDb};port=" . $self->pgPort() . ';host=' . $self->pgSocketPath(),
$self->userGet(), undef,
{AutoCommit => 0, RaiseError => 0, PrintError => 0});
return $self->{db}{$strDb}{hDb} if $self->{db}{$strDb}{hDb};
}
while (!defined($self->{db}{$strDb}{hDb}) && waitMore($oWait));
# Error if unable to connect
if (!defined($self->{db}{$strDb}{hDb}))
{
confess &log(ERROR, "unable to connect to PostgreSQL after ${iTimeout} second(s):\n" . $DBI::errstr, ERROR_DB_CONNECT);
}
}
return $self->{db}{$strDb}{hDb};
}
####################################################################################################################################
# sqlDisconnect
####################################################################################################################################
sub sqlDisconnect
{
my $self = shift;
my $hParam = shift;
foreach my $strDb (keys(%{$self->{db}}))
{
if (defined($$hParam{$strDb}) && $$hParam{$strDb} ne $strDb)
{
next;
}
if (defined($self->{db}{$strDb}{hDb}))
{
$self->{db}{$strDb}{hDb}->disconnect();
undef($self->{db}{$strDb}{hDb});
}
}
}
####################################################################################################################################
# sqlExecute
####################################################################################################################################
sub sqlExecute
{
my $self = shift;
my $strSql = shift;
my $hParam = shift;
# Set defaults
my $bCheckPoint = defined($$hParam{bCheckPoint}) ? $$hParam{bCheckPoint} : false;
my $bCommit = defined($$hParam{bCommit}) ? $$hParam{bCommit} : true;
# Get the db handle
my $hDb = $self->sqlConnect({strDb => $$hParam{strDb}});
# Set autocommit on/off
$hDb->{AutoCommit} = defined($$hParam{bAutoCommit}) ? ($$hParam{bAutoCommit} ? true : false) : false;
# Log and execute the statement
&log(DETAIL, "SQL: ${strSql}");
my $hStatement = $hDb->prepare($strSql);
$hStatement->execute() or
confess &log(ERROR, "Unable to execute: ${strSql}\n" . $DBI::errstr);
$hStatement->finish();
if ($bCommit && !$hDb->{AutoCommit})
{
$self->sqlCommit();
}
# Perform a checkpoint if requested
if ($bCheckPoint)
{
$self->sqlExecute('checkpoint', {bCommit => false, bCheckPoint => false});
}
# Set autocommit off
$hDb->{AutoCommit} = 0;
}
####################################################################################################################################
# sqlSelect
####################################################################################################################################
sub sqlSelect
{
my $self = shift;
my $strSql = shift;
my $hParam = shift;
# Get the db handle
my $hDb = $self->sqlConnect({strDb => $$hParam{strDb}});
# Log and execute the statement
&log(DEBUG, (defined($$hParam{strDb}) ? "DB: $$hParam{strDb}, " : "") . "SQL: ${strSql}");
my $hStatement = $hDb->prepare($strSql);
$hStatement = $hDb->prepare($strSql);
$hStatement->execute() or
confess &log(ERROR, "Unable to execute: ${strSql}\n" . $DBI::errstr);
my @oyRow = $hStatement->fetchrow_array();
$hStatement->finish();
return @oyRow;
}
####################################################################################################################################
# sqlSelectOne
####################################################################################################################################
sub sqlSelectOne
{
my $self = shift;
my $strSql = shift;
my $hParam = shift;
return ($self->sqlSelect($strSql, $hParam))[0];
}
####################################################################################################################################
# sqlSelectOneTest
####################################################################################################################################
sub sqlSelectOneTest
{
my $self = shift;
my $strSql = shift;
my $strExpectedValue = shift;
my $hParam = shift;
# Set defaults
my $iTimeout = defined($$hParam{iTimeout}) ? $$hParam{iTimeout} : HOST_DB_TIMEOUT;
my $lStartTime = time();
my $strActualValue;
do
{
$self->sqlConnect($hParam);
$strActualValue = $self->sqlSelectOne($strSql, $hParam);
if (defined($strActualValue) && $strActualValue eq $strExpectedValue)
{
return;
}
$self->sqlDisconnect();
}
while (defined($iTimeout) && (time() - $lStartTime) <= $iTimeout);
confess &log(
ERROR, "expected value '${strExpectedValue}' from '${strSql}' but actual was '" .
(defined($strActualValue) ? $strActualValue : '[undef]') . "'");
}
####################################################################################################################################
# sqlCommit
####################################################################################################################################
sub sqlCommit
{
my $self = shift;
my $hParam = shift;
my $bCheckPoint = defined($$hParam{bCheckPoint}) ? $$hParam{bCheckPoint} : false;
$self->sqlExecute('commit', {bCommit => false, bCheckPoint => $bCheckPoint});
}
####################################################################################################################################
# sqlWalRotate
####################################################################################################################################
sub sqlWalRotate
{
my $self = shift;
$self->sqlExecute('select pg_switch_' . $self->walId() . '()', {bCommit => false, bCheckPoint => false});
}
####################################################################################################################################
# clusterCreate
#
# Create the PostgreSQL cluster and start it.
####################################################################################################################################
sub clusterCreate
{
my $self = shift;
my $hParam = shift;
# Set defaults
my $strWalPath = defined($$hParam{strWalPath}) ? $$hParam{strWalPath} : $self->dbPath() . '/pg_' . $self->walId();
$self->executeSimple(
$self->pgBinPath() . '/initdb -k' .
($self->pgVersion() >= PG_VERSION_11 ? ' --wal-segsize=1' : '') .
' --' . $self->walId() . "dir=${strWalPath}" . ' --pgdata=' . $self->dbBasePath() . ' --auth=trust');
if (!$self->standby())
{
$self->executeSimple(
"echo 'host replication replicator db-standby trust' >> " . $self->dbBasePath() . '/pg_hba.conf');
}
$self->clusterStart(
{bHotStandby => $$hParam{bHotStandby}, bArchive => $$hParam{bArchive}, bArchiveAlways => $$hParam{bArchiveAlways},
bArchiveInvalid => $$hParam{bArchiveInvalid}});
if (!$self->standby())
{
$self->sqlExecute("create user replicator replication", {bCommit =>true});
}
}
####################################################################################################################################
# clusterStart
#
# Start the PostgreSQL cluster with various test options.
####################################################################################################################################
sub clusterStart
{
my $self = shift;
my $hParam = shift;
# Set defaults
my $bHotStandby = defined($$hParam{bHotStandby}) ? $$hParam{bHotStandby} : false;
my $bArchive = defined($$hParam{bArchive}) ? $$hParam{bArchive} : true;
my $bArchiveAlways = defined($$hParam{bArchiveAlways}) ? $$hParam{bArchiveAlways} : false;
my $bArchiveInvalid = defined($$hParam{bArchiveInvalid}) ? $$hParam{bArchiveInvalid} : false;
my $bArchiveEnabled = defined($$hParam{bArchiveEnabled}) ? $$hParam{bArchiveEnabled} : true;
# Make sure postgres is not running
if (-e $self->dbBasePath() . '/' . DB_FILE_POSTMTRPID)
{
confess DB_FILE_POSTMTRPID . ' exists';
}
# Create the archive command
my $strArchive =
$self->backrestExe() . ' --stanza=' . ($bArchiveInvalid ? 'bogus' : $self->stanza()) .
' --config=' . $self->backrestConfig() . ' archive-push %p';
# Start the cluster
my $strCommand =
$self->pgBinPath() . '/pg_ctl start -o "-c port=' . $self->pgPort() .
($self->pgVersion() < PG_VERSION_95 ? ' -c checkpoint_segments=1' : '');
if ($bArchiveEnabled)
{
if ($self->pgVersion() >= PG_VERSION_95 && $bArchiveAlways)
{
$strCommand .= " -c archive_mode=always";
}
else
{
$strCommand .= " -c archive_mode=on";
}
}
else
{
$strCommand .= " -c archive_mode=off";
}
if ($bArchive)
{
$strCommand .= " -c archive_command='${strArchive}'";
}
else
{
$strCommand .= " -c archive_command=true";
}
$strCommand .= ' -c wal_level=hot_standby -c hot_standby=' . ($bHotStandby ? 'on' : 'off');
# Force parallel mode on to make sure we are disabling it and there are no issues. This is important for testing that 9.6
# works since pg_stop_backup() is marked parallel safe and will error if run in a worker.
if ($self->pgVersion() >= PG_VERSION_96)
{
if ($self->pgVersion() >= PG_VERSION_16)
{
$strCommand .= " -c debug_parallel_query='on'";
}
else
{
$strCommand .= " -c force_parallel_mode='on'";
}
$strCommand .= " -c max_parallel_workers_per_gather=2";
}
$strCommand .=
' -c max_wal_senders=3' .
' -c listen_addresses=\'*\'' .
' -c log_directory=\'' . $self->pgLogPath() . "'" .
' -c log_filename=\'' . basename($self->pgLogFile()) . "'" .
' -c log_rotation_age=0' .
' -c log_rotation_size=0' .
' -c log_error_verbosity=verbose' .
' -c unix_socket_directories=\'' . $self->dbPath() . '\'"' .
' -D ' . $self->dbBasePath() . ' -l ' . $self->pgLogFile() . ' -s';
$self->executeSimple($strCommand);
# Connect user session
$self->sqlConnect();
}
####################################################################################################################################
# clusterStop
#
# Stop the PostgreSQL cluster and optionally check for errors in the server log.
####################################################################################################################################
sub clusterStop
{
my $self = shift;
my $hParam = shift;
# Set defaults
my $bIgnoreLogError = defined($$hParam{bIgnoreLogError}) ? $$hParam{bIgnoreLogError} : false;
my $bStop = defined($hParam->{bStop}) ? $$hParam{bStop} : true;
# Disconnect user session
$self->sqlDisconnect();
# Grep for errors in postgresql.log - this is done first because we want to ignore any errors that happen during shutdown.
if (!$bIgnoreLogError && storageTest()->exists($self->pgLogFile()))
{
$self->executeSimple(
'grep -v "FATAL\: 57P03\: the database system is (starting up|not yet accepting connections|"
"not accepting connections)" ' . $self->pgLogFile() . ' | grep "ERROR\|FATAL"',
{iExpectedExitStatus => 1});
}
# If pg process is running then stop the cluster
if ($bStop && -e $self->dbBasePath() . '/' . DB_FILE_POSTMTRPID)
{
$self->executeSimple($self->pgBinPath() . '/pg_ctl stop -D ' . $self->dbBasePath() . ' -w -s -m fast');
}
# Remove the log file
storageTest()->remove($self->pgLogFile(), {bIgnoreMissing => true});
}
####################################################################################################################################
# clusterRestart
#
# Restart the PostgreSQL cluster.
####################################################################################################################################
sub clusterRestart
{
my $self = shift;
my $hParam = shift;
$self->clusterStop($hParam);
$self->clusterStart($hParam);
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub walId {return shift->pgVersion() >= PG_VERSION_10 ? 'wal' : 'xlog'}
sub pgBinPath {return testRunGet()->pgBinPath()}
sub pgLogFile {return shift->{strPgLogFile}}
sub pgLogPath {return shift->{strPgLogPath}}
sub pgPort {return shift->{iPgPort}}
sub pgSocketPath {return shift->{strPgSocketPath}}
sub pgVersion {return testRunGet()->pgVersion()}
1;

80
test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm
View File

@ -1,80 +0,0 @@
####################################################################################################################################
# GCS Test Host
####################################################################################################################################
package pgBackRestTest::Env::Host::HostGcsTest;
use parent 'pgBackRestTest::Common::HostTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# GCS defaults
####################################################################################################################################
use constant HOST_GCS_BUCKET => 'gcsbucket';
push @EXPORT, qw(HOST_GCS_BUCKET);
use constant HOST_GCS_KEY => 'testkey';
push @EXPORT, qw(HOST_GCS_KEY);
use constant HOST_GCS_KEY_TYPE => 'token';
push @EXPORT, qw(HOST_GCS_KEY_TYPE);
use constant HOST_GCS_PORT => 4443;
push @EXPORT, qw(HOST_GCS_PORT);
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
);
# Create the host
my $strProjectPath = dirname(dirname(abs_path($0)));
my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert";
my $self = $class->SUPER::new(
HOST_GCS, 'test-' . testRunGet()->vmId() . '-' . HOST_GCS, 'fsouza/fake-gcs-server', 'root', undef, undef, undef,
false);
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
1;

86
test/lib/pgBackRestTest/Env/Host/HostS3Test.pm
View File

@ -1,86 +0,0 @@
####################################################################################################################################
# S3 Test Host
####################################################################################################################################
package pgBackRestTest::Env::Host::HostS3Test;
use parent 'pgBackRestTest::Common::HostTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# S3 defaults
####################################################################################################################################
use constant HOST_S3_ACCESS_KEY => 'accessKey1';
push @EXPORT, qw(HOST_S3_ACCESS_KEY);
use constant HOST_S3_ACCESS_SECRET_KEY => 'verySecretKey1';
push @EXPORT, qw(HOST_S3_ACCESS_SECRET_KEY);
use constant HOST_S3_BUCKET => 'pgbackrest-dev';
push @EXPORT, qw(HOST_S3_BUCKET);
use constant HOST_S3_ENDPOINT => 's3.amazonaws.com';
push @EXPORT, qw(HOST_S3_ENDPOINT);
use constant HOST_S3_REGION => 'us-east-1';
push @EXPORT, qw(HOST_S3_REGION);
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
);
# Create the host
my $strProjectPath = dirname(dirname(abs_path($0)));
my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert";
my $self = $class->SUPER::new(
HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', 'minio/minio:RELEASE.2023-09-30T07-02-29Z', 'root',
["${strFakeCertPath}/s3-server.crt:/root/.minio/certs/public.crt:ro",
"${strFakeCertPath}/s3-server.key:/root/.minio/certs/private.key:ro"],
'-e MINIO_REGION=' . HOST_S3_REGION . ' -e MINIO_DOMAIN=' . HOST_S3_ENDPOINT . ' -e MINIO_BROWSER=off' .
' -e MINIO_ACCESS_KEY=' . HOST_S3_ACCESS_KEY . ' -e MINIO_SECRET_KEY=' . HOST_S3_ACCESS_SECRET_KEY,
'server /data --address :443', false);
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
1;

72
test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm
View File

@ -1,72 +0,0 @@
####################################################################################################################################
# SFTP Test Host
####################################################################################################################################
package pgBackRestTest::Env::Host::HostSftpTest;
use parent 'pgBackRestTest::Common::HostTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# SFTP defaults
####################################################################################################################################
use constant HOST_SFTP_ACCOUNT => TEST_USER;
push @EXPORT, qw(HOST_SFTP_ACCOUNT);
use constant HOST_SFTP_HOSTKEY_HASH_TYPE => 'sha1';
push @EXPORT, qw(HOST_SFTP_HOSTKEY_HASH_TYPE);
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
);
# Create the host
my $self = $class->SUPER::new(
HOST_SFTP, 'test-' . testRunGet()->vmId() . '-' . HOST_SFTP, containerRepo() . ':' . testRunGet()->vm() . '-test', 'root');
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
1;

543
test/lib/pgBackRestTest/Env/HostEnvTest.pm
View File

@ -1,543 +0,0 @@
####################################################################################################################################
# FullCommonTest.pm - Common code for backup tests
####################################################################################################################################
package pgBackRestTest::Env::HostEnvTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Digest::SHA qw(sha1_hex);
use Exporter qw(import);
our @EXPORT = qw();
use Storable qw(dclone);
use pgBackRestDoc::Common::Log;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::StorageBase;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Env::ArchiveInfo;
use pgBackRestTest::Env::Host::HostAzureTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Host::HostDbCommonTest;
use pgBackRestTest::Env::Host::HostDbTest;
use pgBackRestTest::Env::Host::HostDbSyntheticTest;
use pgBackRestTest::Env::Host::HostGcsTest;
use pgBackRestTest::Env::Host::HostS3Test;
####################################################################################################################################
# Constants
####################################################################################################################################
use constant ENCRYPTION_KEY_ARCHIVE => 'archive';
push @EXPORT, qw(ENCRYPTION_KEY_ARCHIVE);
use constant ENCRYPTION_KEY_MANIFEST => 'manifest';
push @EXPORT, qw(ENCRYPTION_KEY_MANIFEST);
use constant ENCRYPTION_KEY_BACKUPSET => 'backupset';
push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET);
####################################################################################################################################
# setup
####################################################################################################################################
sub setup
{
my $self = shift;
my $bSynthetic = shift;
my $oConfigParam = shift;
# Start object server first since it takes the longest
#-------------------------------------------------------------------------------------------------------------------------------
my $oHostObject;
if ($oConfigParam->{strStorage} eq S3)
{
$oHostObject = new pgBackRestTest::Env::Host::HostS3Test();
}
elsif ($oConfigParam->{strStorage} eq AZURE)
{
$oHostObject = new pgBackRestTest::Env::Host::HostAzureTest();
}
elsif ($oConfigParam->{strStorage} eq GCS)
{
$oHostObject = new pgBackRestTest::Env::Host::HostGcsTest();
}
elsif ($oConfigParam->{strStorage} eq SFTP)
{
$oHostObject = new pgBackRestTest::Env::Host::HostSftpTest();
}
# Get host group
my $oHostGroup = hostGroupGet();
# Create the backup host
my $strBackupDestination;
my $bHostBackup = defined($$oConfigParam{bHostBackup}) ? $$oConfigParam{bHostBackup} : false;
my $oHostBackup = undef;
my $bRepoEncrypt = defined($$oConfigParam{bRepoEncrypt}) ? $$oConfigParam{bRepoEncrypt} : false;
if ($bHostBackup)
{
$strBackupDestination = defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_BACKUP;
$oHostBackup = new pgBackRestTest::Env::Host::HostBackupTest(
{strBackupDestination => $strBackupDestination, bSynthetic => $bSynthetic,
bRepoLocal => $oConfigParam->{strStorage} eq POSIX, bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}});
$oHostGroup->hostAdd($oHostBackup);
}
else
{
$strBackupDestination =
defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_DB_PRIMARY;
}
# Create the db-primary host
my $oHostDbPrimary = undef;
if ($bSynthetic)
{
$oHostDbPrimary = new pgBackRestTest::Env::Host::HostDbSyntheticTest(
{strBackupDestination => $strBackupDestination,
bRepoLocal => $oConfigParam->{strStorage} eq POSIX, bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}});
}
else
{
$oHostDbPrimary = new pgBackRestTest::Env::Host::HostDbTest(
{strBackupDestination => $strBackupDestination, bRepoLocal => $oConfigParam->{strStorage} eq POSIX,
bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}});
}
$oHostGroup->hostAdd($oHostDbPrimary);
# Create the db-standby host
my $oHostDbStandby = undef;
if (defined($$oConfigParam{bStandby}) && $$oConfigParam{bStandby})
{
$oHostDbStandby = new pgBackRestTest::Env::Host::HostDbTest(
{strBackupDestination => $strBackupDestination, bStandby => true, bRepoLocal => $oConfigParam->{strStorage} eq POSIX,
bTls => $oConfigParam->{bTls}});
$oHostGroup->hostAdd($oHostDbStandby);
}
# Finalize object server
#-------------------------------------------------------------------------------------------------------------------------------
if ($oConfigParam->{strStorage} eq S3)
{
$oHostGroup->hostAdd($oHostObject, {rstryHostName => ['pgbackrest-dev.s3.amazonaws.com', 's3.amazonaws.com']});
}
elsif ($oConfigParam->{strStorage} eq AZURE || $oConfigParam->{strStorage} eq GCS || $oConfigParam->{strStorage} eq SFTP)
{
$oHostGroup->hostAdd($oHostObject);
}
# Create db-primary config
$oHostDbPrimary->configCreate({
bTls => $oConfigParam->{bTls},
strBackupSource => $$oConfigParam{strBackupSource},
strCompressType => $$oConfigParam{strCompressType},
bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink},
bArchiveAsync => $$oConfigParam{bArchiveAsync},
strStorage => $oConfigParam->{strStorage},
iRepoTotal => $oConfigParam->{iRepoTotal},
bBundle => $oConfigParam->{bBundle},
bBlockIncr => $oConfigParam->{bBlockIncr}});
# Create backup config if backup host exists
if (defined($oHostBackup))
{
$oHostBackup->configCreate({
bTls => $oConfigParam->{bTls},
strCompressType => $$oConfigParam{strCompressType},
bHardlink => $$oConfigParam{bHardLink},
strStorage => $oConfigParam->{strStorage},
iRepoTotal => $oConfigParam->{iRepoTotal},
bBundle => $oConfigParam->{bBundle},
bBlockIncr => $oConfigParam->{bBlockIncr}});
}
# If backup host is not defined set it to db-primary
else
{
$oHostBackup = $strBackupDestination eq HOST_DB_PRIMARY || $strBackupDestination eq HOST_SFTP ? $oHostDbPrimary :
$oHostDbStandby;
}
storageRepoCommandSet(
$self->backrestExeHelper() .
' --config=' . $oHostBackup->backrestConfig() . ' --stanza=' . $self->stanza() . ' --log-level-console=off' .
' --log-level-stderr=error' .
($oConfigParam->{strStorage} ne POSIX ?
($oConfigParam->{strStorage} ne SFTP ? " --no-repo1-storage-verify-tls" : '') .
" --repo1-$oConfigParam->{strStorage}-" .
($oConfigParam->{strStorage} eq GCS ? 'endpoint' : 'host') . "=" . $oHostObject->ipGet() : '') .
($oConfigParam->{strStorage} eq GCS ? ':' . HOST_GCS_PORT : ''),
$oConfigParam->{strStorage} eq POSIX ? STORAGE_POSIX : STORAGE_OBJECT);
# Create db-standby config
if (defined($oHostDbStandby))
{
$oHostDbStandby->configCreate({
bTls => $oConfigParam->{bTls},
strBackupSource => $$oConfigParam{strBackupSource},
strCompressType => $$oConfigParam{strCompressType},
bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink},
bArchiveAsync => $$oConfigParam{bArchiveAsync},
strStorage => $oConfigParam->{strStorage},
iRepoTotal => $oConfigParam->{iRepoTotal},
bBundle => $oConfigParam->{bBundle},
bBlockIncr => $oConfigParam->{bBlockIncr}});
}
# Create object storage
if (defined($oHostObject))
{
storageRepo()->create();
}
return $oHostDbPrimary, $oHostDbStandby, $oHostBackup;
}
####################################################################################################################################
# Generate database system id for the db version
####################################################################################################################################
sub dbSysId
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPgVersion,
) =
logDebugParam
(
__PACKAGE__ . '->dbSysId', \@_,
{name => 'strPgVersion', trace => true},
);
return (1000000000000000000 + ($strPgVersion * 10));
}
####################################################################################################################################
# Get database catalog version for the db version
####################################################################################################################################
sub dbCatalogVersion
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPgVersion,
) =
logDebugParam
(
__PACKAGE__ . '->sysId', \@_,
{name => 'strPgVersion', trace => true},
);
my $hCatalogVersion =
{
&PG_VERSION_94 => 201409291,
&PG_VERSION_95 => 201510051,
&PG_VERSION_96 => 201608131,
&PG_VERSION_10 => 201707211,
&PG_VERSION_11 => 201806231,
&PG_VERSION_12 => 201909212,
&PG_VERSION_13 => 202007201,
&PG_VERSION_14 => 202105121,
&PG_VERSION_15 => 202209061,
&PG_VERSION_16 => 202307071,
};
if (!defined($hCatalogVersion->{$strPgVersion}))
{
confess &log(ASSERT, "no catalog version defined for pg version ${strPgVersion}");
}
return $hCatalogVersion->{$strPgVersion};
}
####################################################################################################################################
# Generate control file content
####################################################################################################################################
sub controlGenerateContent
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPgVersion,
) =
logDebugParam
(
__PACKAGE__ . '->controlGenerateContent', \@_,
{name => 'strPgVersion', trace => true},
);
my $hControlContent =
{
32 =>
{
&PG_VERSION_94 =>
"5e0064a7b3b6e00dae0300000b43010c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000002000000000000000000000000000010000000000000000000000000000000000000000" .
"000000006b0756c8",
&PG_VERSION_95 =>
"5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
"000000000000000000000000000000003bfe413a",
&PG_VERSION_96 =>
"600064a7b3b6e00dc0030000c34b040c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
"000000000000000000000000000000005d135da6",
&PG_VERSION_10 =>
"640064a7b3b6e00dea030000cbce050c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8556c34",
},
64 =>
{
&PG_VERSION_94 =>
"5e0064a7b3b6e00dae0300000b43010c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
"00000000000000000000000000000000ee6cf996",
&PG_VERSION_95 =>
"5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
"0000000000000001000000000000000000000000000000000000000000000000381ec2de",
&PG_VERSION_96 =>
"600064a7b3b6e00dc0030000c34b040c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
"00000000000000010000000000000000000000000000000000000000000000002d96a4c0",
&PG_VERSION_10 =>
"640064a7b3b6e00dea030000cbce050c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
"0000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"00000000000000008d543cdf",
},
};
my $strControlContent = $hControlContent->{$self->archBits()}{$strPgVersion};
if (!defined($strControlContent))
{
confess &log(ASSERT, "no control content defined for pg version ${strPgVersion}");
}
my $tControlContent = '';
for (my $iIdx = 0; $iIdx < length($strControlContent) / 2; $iIdx++)
{
my $iChar = hex(substr($strControlContent, $iIdx * 2, 2));
$tControlContent .= pack('C', $iChar);
}
# Pad bytes
for (my $iIdx = length($tControlContent); $iIdx < 8192; $iIdx++)
{
$tControlContent .= pack('C', 0);
}
return \$tControlContent;
}
####################################################################################################################################
# Generate control file and write to disk
####################################################################################################################################
sub controlGenerate
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbPath,
$strPgVersion,
) =
logDebugParam
(
__PACKAGE__ . '->controlGenerate', \@_,
{name => 'strDbPath', trace => true},
{name => 'strPgVersion', trace => true},
);
storageTest()->put("${strDbPath}/global/pg_control", $self->controlGenerateContent($strPgVersion));
}
####################################################################################################################################
# walSegment
#
# Generate name of WAL segment from component parts.
####################################################################################################################################
sub walSegment
{
my $self = shift;
my $iTimeline = shift;
my $iMajor = shift;
my $iMinor = shift;
return uc(sprintf('%08x%08x%08x', $iTimeline, $iMajor, $iMinor));
}
####################################################################################################################################
# Generate WAL file content
####################################################################################################################################
sub walGenerateContent
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPgVersion,
$iSourceNo,
) =
logDebugParam
(
__PACKAGE__ . '->walGenerateContent', \@_,
{name => 'strPgVersion', trace => true},
{name => 'iSourceNo', optional => true, default => 1, trace => true},
);
# Get WAL magic for the PG version
my $hWalMagic =
{
&PG_VERSION_94 => hex('0xD07E'),
&PG_VERSION_95 => hex('0xD087'),
&PG_VERSION_96 => hex('0xD093'),
&PG_VERSION_10 => hex('0xD097'),
&PG_VERSION_11 => hex('0xD098'),
&PG_VERSION_12 => hex('0xD101'),
&PG_VERSION_13 => hex('0xD106'),
};
my $tWalContent = pack('S', $hWalMagic->{$strPgVersion});
# Indicate that the long header is present
$tWalContent .= pack('S', 2);
# Add junk (H for header) for the bytes that won't be read by the tests
my $iOffset = 12 + (testRunGet()->archBits() / 8);
$tWalContent .= ('H' x $iOffset);
# Add the system identifier
$tWalContent .= pack('Q', $self->dbSysId($strPgVersion));
# Add segment size
$tWalContent .= pack('L', PG_WAL_SEGMENT_SIZE);
# Add the source number to produce WAL segments with different checksums
$tWalContent .= pack('S', $iSourceNo);
# Pad out to the required size (B for body)
$tWalContent .= ('B' x (PG_WAL_SEGMENT_SIZE - length($tWalContent)));
return \$tWalContent;
}
####################################################################################################################################
# Generate WAL file content checksum
####################################################################################################################################
sub walGenerateContentChecksum
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPgVersion,
$hParam,
) =
logDebugParam
(
__PACKAGE__ . '->walGenerateContent', \@_,
{name => 'strPgVersion', trace => true},
{name => 'hParam', required => false, trace => true},
);
return sha1_hex(${$self->walGenerateContent($strPgVersion, $hParam)});
}
####################################################################################################################################
# walGenerate
#
# Generate a WAL segment and ready file for testing.
####################################################################################################################################
sub walGenerate
{
my $self = shift;
my $strWalPath = shift;
my $strPgVersion = shift;
my $iSourceNo = shift;
my $strWalSegment = shift;
my $bPartial = shift;
my $bChecksum = shift;
my $bReady = shift;
my $rtWalContent = $self->walGenerateContent($strPgVersion, {iSourceNo => $iSourceNo});
my $strWalFile =
"${strWalPath}/${strWalSegment}" . ($bChecksum ? '-' . sha1_hex($rtWalContent) : '') .
(defined($bPartial) && $bPartial ? '.partial' : '');
# Put the WAL segment and the ready file
storageTest()->put($strWalFile, $rtWalContent);
if (!defined($bReady) || $bReady)
{
storageTest()->put("${strWalPath}/archive_status/${strWalSegment}.ready");
}
return $strWalFile;
}
####################################################################################################################################
# walRemove
#
# Remove WAL file and ready file.
####################################################################################################################################
sub walRemove
{
my $self = shift;
my $strWalPath = shift;
my $strWalFile = shift;
storageTest()->remove("$self->{strWalPath}/${strWalFile}");
storageTest()->remove("$self->{strWalPath}/archive_status/${strWalFile}.ready");
}
1;

32
test/lib/pgBackRestTest/Env/InfoCommon.pm
View File

@ -1,32 +0,0 @@
####################################################################################################################################
# INFO MODULE
# Constants, variables and functions common to the info files
####################################################################################################################################
package pgBackRestTest::Env::InfoCommon;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
####################################################################################################################################
# DB section constants
####################################################################################################################################
use constant INFO_BACKUP_SECTION_DB => 'db';
push @EXPORT, qw(INFO_BACKUP_SECTION_DB);
use constant INFO_BACKUP_SECTION_DB_HISTORY => INFO_BACKUP_SECTION_DB . ':history';
push @EXPORT, qw(INFO_BACKUP_SECTION_DB_HISTORY);
####################################################################################################################################
# History section constants
####################################################################################################################################
use constant INFO_HISTORY_ID => 'id';
push @EXPORT, qw(INFO_HISTORY_ID);
use constant INFO_DB_VERSION => 'version';
push @EXPORT, qw(INFO_DB_VERSION);
use constant INFO_SYSTEM_ID => 'system-id';
push @EXPORT, qw(INFO_SYSTEM_ID);
1;

1446
test/lib/pgBackRestTest/Env/Manifest.pm
View File

File diff suppressed because it is too large Load Diff

747
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
View File

@ -1,747 +0,0 @@
####################################################################################################################################
# Test All Commands On PostgreSQL Clusters
####################################################################################################################################
package pgBackRestTest::Module::Real::RealAllTest;
use parent 'pgBackRestTest::Env::HostEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(dirname);
use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;
use pgBackRestDoc::ProjectInfo;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::VmTest;
use pgBackRestTest::Common::Storage;
use pgBackRestTest::Common::StoragePosix;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;
use pgBackRestTest::Env::ArchiveInfo;
use pgBackRestTest::Env::BackupInfo;
use pgBackRestTest::Env::InfoCommon;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostDbTest;
use pgBackRestTest::Env::Host::HostDbTest;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Env::Manifest;
####################################################################################################################################
# Backup advisory lock
####################################################################################################################################
use constant DB_BACKUP_ADVISORY_LOCK => '12340078987004321';
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
foreach my $rhRun
(
{pg => '9.4', dst => 'db-standby', tls => 0, stg => POSIX, enc => 1, cmp => LZ4, rt => 1, bnd => 1, bi => 0},
{pg => '9.5', dst => 'backup', tls => 1, stg => GCS, enc => 0, cmp => BZ2, rt => 1, bnd => 0, bi => 1},
{pg => '9.6', dst => 'backup', tls => 0, stg => POSIX, enc => 0, cmp => NONE, rt => 2, bnd => 1, bi => 1},
{pg => '10', dst => 'sftp-srvr', tls => 0, stg => SFTP, enc => 1, cmp => GZ, rt => 1, bnd => 1, bi => 0},
{pg => '11', dst => 'backup', tls => 1, stg => AZURE, enc => 0, cmp => ZST, rt => 2, bnd => 0, bi => 0},
{pg => '12', dst => 'backup', tls => 0, stg => S3, enc => 1, cmp => LZ4, rt => 1, bnd => 0, bi => 1},
{pg => '13', dst => 'db-standby', tls => 1, stg => GCS, enc => 0, cmp => ZST, rt => 1, bnd => 1, bi => 1},
{pg => '14', dst => 'sftp-srvr', tls => 0, stg => SFTP, enc => 0, cmp => LZ4, rt => 1, bnd => 1, bi => 0},
{pg => '15', dst => 'db-standby', tls => 0, stg => AZURE, enc => 0, cmp => NONE, rt => 2, bnd => 1, bi => 1},
{pg => '16', dst => 'backup', tls => 0, stg => S3, enc => 1, cmp => NONE, rt => 1, bnd => 0, bi => 0},
)
{
# Only run tests for this pg version
next if ($rhRun->{pg} ne $self->pgVersion());
# Get run parameters
my $bHostBackup = $rhRun->{dst} eq HOST_BACKUP ? true : false;
my $bTls = $rhRun->{tls};
my $strBackupDestination = $rhRun->{dst};
my $strStorage = $rhRun->{stg};
my $bRepoEncrypt = $rhRun->{enc};
my $strCompressType = $rhRun->{cmp};
my $iRepoTotal = $rhRun->{rt};
my $bBundle = $rhRun->{bnd};
my $bBlockIncr = $rhRun->{bi};
# Some tests are not version specific so only run them on a single version of PostgreSQL
my $bNonVersionSpecific = $self->pgVersion() eq PG_VERSION_96;
# Increment the run, log, and decide whether this unit test should be run
next if !$self->begin(
"bkp ${bHostBackup}, tls ${bTls}, dst ${strBackupDestination}, cmp ${strCompressType}, storage ${strStorage}" .
", enc ${bRepoEncrypt}, bi ${bBlockIncr}");
# Create hosts, file object, and config
my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup(
false,
{bHostBackup => $bHostBackup, bStandby => true, bTls => $bTls, strBackupDestination => $strBackupDestination,
strCompressType => $strCompressType, bArchiveAsync => false, strStorage => $strStorage,
bRepoEncrypt => $bRepoEncrypt, iRepoTotal => $iRepoTotal, bBundle => $bBundle, bBlockIncr => $bBlockIncr});
# Some commands will fail because of the bogus host created when a standby is present. These options reset the bogus host
# so it won't interfere with commands that won't tolerate a connection failure.
my $strBogusReset = $oHostBackup->bogusHost() ?
' --reset-pg2-host --reset-pg2-host-type --reset-pg2-host-cmd --reset-pg2-host-config --reset-pg2-host-user' .
' --reset-pg2-path' :
'';
# If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing
# without slowing down the other tests too much.
if ($strStorage eq S3)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
$oHostDbPrimary->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
}
$oHostDbPrimary->clusterCreate();
# Create the stanza
$oHostDbPrimary->stanzaCreate('main create stanza info files');
# Get passphrase to access the Manifest file from backup.info - returns undefined if repo not encrypted
my $strCipherPass =
(new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()))->cipherPassSub();
# Create a manifest with the pg version to get version-specific paths
my $oManifest = new pgBackRestTest::Env::Manifest(BOGUS, {bLoad => false, strDbVersion => $self->pgVersion(),
iDbCatalogVersion => $self->dbCatalogVersion($self->pgVersion()),
strCipherPass => $strCipherPass, strCipherPassSub => $bRepoEncrypt ? ENCRYPTION_KEY_BACKUPSET : undef});
# Static backup parameters
my $fTestDelay = 1;
# Restore test string
my $strDefaultMessage = 'default';
my $strFullMessage = 'full';
my $strStandbyMessage = 'standby';
my $strIncrMessage = 'incr';
my $strTimeMessage = 'time';
my $strXidMessage = 'xid';
my $strNameMessage = 'name';
my $strTimelineMessage = 'timeline';
# Create two new databases
$oHostDbPrimary->sqlExecute('create database test1', {bAutoCommit => true});
$oHostDbPrimary->sqlExecute('create database test2', {bAutoCommit => true});
# ??? Removed temporarily until manifest build can be brought back into the check command
# Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check
# --------------------------------------------------------------------------------------------------------------------------
# my $strDir = $oHostDbPrimary->dbBasePath() . '/rootreaddir';
# executeTest('sudo mkdir ' . $strDir);
# executeTest("sudo chown root:root ${strDir}");
# executeTest("sudo chmod 400 ${strDir}");
#
# $strComment = 'confirm primary manifest->build executed';
# $oHostDbPrimary->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN});
# executeTest("sudo rmdir ${strDir}");
# --------------------------------------------------------------------------------------------------------------------------
my $strComment = 'verify check command runs successfully';
$oHostDbPrimary->check($strComment, {iTimeout => 10, bStanza => false});
# Also run check on the backup host when present
if ($bHostBackup)
{
$oHostBackup->check($strComment, {iTimeout => 10, strOptionalParam => $strBogusReset});
}
# Restart the cluster ignoring any errors in the postgresql log
$oHostDbPrimary->clusterRestart({bIgnoreLogError => true});
# Full backup
#---------------------------------------------------------------------------------------------------------------------------
# Create the table where test messages will be stored
$oHostDbPrimary->sqlExecute("create table test (message text not null)");
$oHostDbPrimary->sqlWalRotate();
$oHostDbPrimary->sqlExecute("insert into test values ('$strDefaultMessage')");
# Acquire the backup advisory lock so it looks like a backup is running
if (!$oHostDbPrimary->sqlSelectOne('select pg_try_advisory_lock(' . DB_BACKUP_ADVISORY_LOCK . ')'))
{
confess 'unable to acquire advisory lock for testing';
}
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'fail on backup lock exists', {iExpectedExitStatus => ERROR_LOCK_ACQUIRE});
# Release the backup advisory lock so the next backup will succeed
if (!$oHostDbPrimary->sqlSelectOne('select pg_advisory_unlock(' . DB_BACKUP_ADVISORY_LOCK . ')'))
{
confess 'unable to release advisory lock';
}
$oHostDbPrimary->sqlExecute("update test set message = '$strFullMessage'");
# Required to set hint bits to be sent to the standby to make the heap match on both sides
$oHostDbPrimary->sqlSelectOneTest('select message from test', $strFullMessage);
# Backup to repo1
my $strFullBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'repo1',
{strOptionalParam => ' --buffer-size=16384'});
# Backup to repo2 if it exists
if ($iRepoTotal == 2)
{
$oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_FULL, 'repo2', {iRepo => 2});
}
# Make a new backup with expire-auto disabled then run the expire command and compare backup numbers to ensure that expire
# was really disabled. This test is not version specific so is run on only one version.
#---------------------------------------------------------------------------------------------------------------------------
if ($bNonVersionSpecific)
{
my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath());
push(my @backupLst1, $oBackupInfo->list());
$strFullBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'with disabled expire-auto',
{strOptionalParam => ' --repo1-retention-full='.scalar(@backupLst1). ' --no-expire-auto'});
$oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath());
push(my @backupLst2, $oBackupInfo->list());
&log(INFO, " run the expire command");
$oHostBackup->expire({iRetentionFull => scalar(@backupLst1)});
$oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath());
push(my @backupLst3, $oBackupInfo->list());
unless (scalar(@backupLst2) == scalar(@backupLst1) + 1 && scalar(@backupLst1) == scalar(@backupLst3))
{
confess "expire-auto option didn't work as expected";
}
}
# Enabled async archiving
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'archive-async' => 'y'}});
# Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it down
# enough to make it evident that the async process is working.
if ($strCompressType ne NONE && $strBackupDestination eq HOST_BACKUP)
{
&log(INFO, ' multiple wal switches to exercise async archiving');
$oHostDbPrimary->sqlExecute("create table wal_activity (id int)");
$oHostDbPrimary->sqlWalRotate();
$oHostDbPrimary->sqlExecute("insert into wal_activity values (1)");
$oHostDbPrimary->sqlWalRotate();
$oHostDbPrimary->sqlExecute("insert into wal_activity values (2)");
$oHostDbPrimary->sqlWalRotate();
$oHostDbPrimary->sqlExecute("insert into wal_activity values (3)");
$oHostDbPrimary->sqlWalRotate();
$oHostDbPrimary->sqlExecute("insert into wal_activity values (4)");
$oHostDbPrimary->sqlWalRotate();
}
# Setup replica
#---------------------------------------------------------------------------------------------------------------------------
my %oRemapHash;
$oRemapHash{&MANIFEST_TARGET_PGDATA} = $oHostDbStandby->dbBasePath();
$oHostDbStandby->linkRemap($oManifest->walPath(), $oHostDbStandby->dbPath() . '/' . $oManifest->walPath());
$oHostDbStandby->restore(
'restore backup on replica', 'latest',
{rhRemapHash => \%oRemapHash, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY,
strOptionalParam =>
' --recovery-option="primary_conninfo=host=' . HOST_DB_PRIMARY .
' port=' . $oHostDbPrimary->pgPort() . ' user=replicator"'});
$oHostDbStandby->clusterStart({bHotStandby => true});
# Make sure streaming replication is on
$oHostDbPrimary->sqlSelectOneTest(
"select client_addr || '-' || state from pg_stat_replication", $oHostDbStandby->ipGet() . '/32-streaming');
# Check that the cluster was restored properly
$oHostDbStandby->sqlSelectOneTest('select message from test', $strFullMessage);
# Update message for standby
$oHostDbPrimary->sqlExecute("update test set message = '$strStandbyMessage'");
if (!$bTls)
{
# If there is only a primary and a replica and the replica is the backup destination, then if pg2-host and
# pg256-host are BOGUS, confirm failure to reach the primary
if (!$bHostBackup && $strBackupDestination eq HOST_DB_STANDBY)
{
my $strStandbyBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to reach primary',
{bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg256-host=' . BOGUS});
}
else
{
my $strStandbyBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to access at least one standby',
{bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg256-host=' . BOGUS});
}
}
my $strStandbyBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby',
{bStandby => true, iExpectedExitStatus => undef, strOptionalParam => '--repo1-retention-full=1'});
$strFullBackup = $strStandbyBackup;
# ??? Removed temporarily until manifest build can be brought back into the check command
# # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check
# my $strDir = $oHostDbStandby->dbBasePath() . '/rootreaddir';
# executeTest('sudo mkdir ' . $strDir);
# executeTest("sudo chown root:root ${strDir}");
# executeTest("sudo chmod 400 ${strDir}");
#
# my $strComment = 'confirm standby manifest->build executed';
#
# # If there is an invalid host, the final error returned from check will be the inability to resolve the name which is
# # an open error instead of a read error
# if (!$oHostDbStandby->bogusHost())
# {
# $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN});
# }
# else
# {
# $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_READ});
# }
#
# # Remove the directory in pg_data location that is only readable by root
# executeTest("sudo rmdir ${strDir}");
# Confirm the check command runs without error on a standby (when a bogus host is not configured)
$oHostDbStandby->check('verify check command on standby', {strOptionalParam => $strBogusReset});
# Shutdown the standby before creating tablespaces (this will error since paths are different)
$oHostDbStandby->clusterStop({bIgnoreLogError => true});
my $strAdhocBackup;
# Execute stop and make sure the backup fails
#---------------------------------------------------------------------------------------------------------------------------
# Restart the cluster to check for any errors before continuing since the stop tests will definitely create errors and the
# logs will need to be deleted to avoid causing issues further down the line. This test is not version specific so is run on
# only one version.
if ($bNonVersionSpecific)
{
confess "test must be performed on posix storage" if $strStorage ne POSIX;
$oHostDbPrimary->clusterRestart();
# Add backup for adhoc expire
$strAdhocBackup = $oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_DIFF, 'backup for adhoc expire');
$oHostDbPrimary->stop();
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'attempt backup when stopped',
{iExpectedExitStatus => $oHostBackup == $oHostDbPrimary ? ERROR_STOP : ERROR_DB_CONNECT});
$oHostDbPrimary->start();
}
# Setup the time targets
#---------------------------------------------------------------------------------------------------------------------------
# If the tests are running quickly then the time target might end up the same as the end time of the prior full backup. That
# means restore auto-select will not pick it as a candidate and restore the last backup instead causing the restore compare
# to fail. So, sleep one second.
sleep(1);
$oHostDbPrimary->sqlExecute("update test set message = '$strTimeMessage'");
$oHostDbPrimary->sqlWalRotate();
my $strTimeTarget = $oHostDbPrimary->sqlSelectOne("select current_timestamp");
&log(INFO, " time target is ${strTimeTarget}");
# Incr backup - fail on archive_mode=always when version >= 9.5
#---------------------------------------------------------------------------------------------------------------------------
if ($oHostDbPrimary->pgVersion() >= PG_VERSION_95)
{
# Set archive_mode=always
$oHostDbPrimary->clusterRestart({bArchiveAlways => true});
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on archive_mode=always', {iExpectedExitStatus => ERROR_FEATURE_NOT_SUPPORTED});
# Reset the cluster to a normal state so the next test will work
$oHostDbPrimary->clusterRestart();
}
# Incr backup
#---------------------------------------------------------------------------------------------------------------------------
# Create a tablespace directory
storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700', bCreateParent => true});
# Also create it on the standby so replay won't fail
if (defined($oHostDbStandby))
{
storageTest()->pathCreate($oHostDbStandby->tablespacePath(1), {strMode => '0700', bCreateParent => true});
}
$oHostDbPrimary->sqlExecute(
"create tablespace ts1 location '" . $oHostDbPrimary->tablespacePath(1) . "'", {bAutoCommit => true});
$oHostDbPrimary->sqlExecute("alter table test set tablespace ts1");
# Create a table in the tablespace that will not be modified again to be sure it does get full page writes in the WAL later
$oHostDbPrimary->sqlExecute("create table test_exists (id int) tablespace ts1", {bCommit => true, bCheckPoint => true});
# Create a table in the tablespace
$oHostDbPrimary->sqlExecute("create table test_remove (id int)");
$oHostDbPrimary->sqlWalRotate();
$oHostDbPrimary->sqlExecute("update test set message = '$strDefaultMessage'");
$oHostDbPrimary->sqlWalRotate();
# Create a database in the tablespace and a table to check
$oHostDbPrimary->sqlExecute("create database test3 with tablespace ts1", {bAutoCommit => true});
$oHostDbPrimary->sqlExecute(
'create table test3_exists (id int);' .
'insert into test3_exists values (1);',
{strDb => 'test3', bAutoCommit => true});
# Create a table in test1 to check - test1 will not be restored
$oHostDbPrimary->sqlExecute(
'create table test1_zeroed (id int);' .
'insert into test1_zeroed values (1);',
{strDb => 'test1', bAutoCommit => true});
# Start a backup so the next backup has to restart it. This test is not required for PostgreSQL >= 9.6 since backups are run
# in non-exclusive mode.
if ($oHostDbPrimary->pgVersion() < PG_VERSION_96)
{
$oHostDbPrimary->sqlSelectOne("select pg_start_backup('test backup that will cause an error', true)");
# Verify that an error is returned if the backup is already running
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on backup already running', {iExpectedExitStatus => ERROR_DB_QUERY});
# Restart the cluster ignoring any errors in the postgresql log
$oHostDbPrimary->clusterRestart({bIgnoreLogError => true});
# Start a new backup to make the next test restarts it
$oHostDbPrimary->sqlSelectOne("select pg_start_backup('test backup that will be restarted', true)");
}
if (defined($strAdhocBackup))
{
# Adhoc expire the latest backup - no other tests should be affected
$oHostBackup->expire({strOptionalParam => '--set=' . $strAdhocBackup});
}
# Drop a table
$oHostDbPrimary->sqlExecute('drop table test_remove');
$oHostDbPrimary->sqlWalRotate();
$oHostDbPrimary->sqlExecute("update test set message = '$strIncrMessage'", {bCommit => true});
# Exercise --delta checksum option
my $strIncrBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'delta',
{strOptionalParam => '--stop-auto --buffer-size=32768 --delta', iRepo => $iRepoTotal});
# Ensure the check command runs properly with a tablespace
$oHostBackup->check( 'check command with tablespace', {iTimeout => 10, strOptionalParam => $strBogusReset});
# Setup the xid target
#---------------------------------------------------------------------------------------------------------------------------
my $strXidTarget = undef;
$oHostDbPrimary->sqlExecute("update test set message = '$strXidMessage'", {bCommit => false});
$oHostDbPrimary->sqlWalRotate();
$strXidTarget = $oHostDbPrimary->sqlSelectOne("select txid_current()");
$oHostDbPrimary->sqlCommit();
&log(INFO, " xid target is ${strXidTarget}");
# Setup the name target
#---------------------------------------------------------------------------------------------------------------------------
my $strNameTarget = 'backrest';
$oHostDbPrimary->sqlExecute("update test set message = '$strNameMessage'", {bCommit => true});
$oHostDbPrimary->sqlWalRotate();
$oHostDbPrimary->sqlExecute("select pg_create_restore_point('${strNameTarget}')");
&log(INFO, " name target is ${strNameTarget}");
# Create a table and data in database test2
#---------------------------------------------------------------------------------------------------------------------------
# Initialize variables for SHA1 and path of the pg_filenode.map for the database that will not be restored
my $strDb1TablePath;
my $strDb1TableSha1;
$oHostDbPrimary->sqlExecute(
'create table test (id int);' .
'insert into test values (1);' .
'create table test_ts1 (id int) tablespace ts1;' .
'insert into test_ts1 values (2);',
{strDb => 'test2', bAutoCommit => true});
$oHostDbPrimary->sqlWalRotate();
# Get the SHA1 and path of the table for the database that will not be restored
$strDb1TablePath = $oHostDbPrimary->dbBasePath(). "/base/" .
$oHostDbPrimary->sqlSelectOne("select oid from pg_database where datname='test1'") . "/" .
$oHostDbPrimary->sqlSelectOne("select relfilenode from pg_class where relname='test1_zeroed'", {strDb => 'test1'});
$strDb1TableSha1 = storageTest()->hashSize($strDb1TablePath);
# Restore (type = default)
#---------------------------------------------------------------------------------------------------------------------------
# Expect failure because pg (appears to be) running
$oHostDbPrimary->restore('pg running', 'latest', {iExpectedExitStatus => ERROR_PG_RUNNING});
$oHostDbPrimary->clusterStop();
# Expect failure because db path is not empty
$oHostDbPrimary->restore('path not empty', 'latest', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
# Drop and recreate db path
testPathRemove($oHostDbPrimary->dbBasePath());
storageTest()->pathCreate($oHostDbPrimary->dbBasePath(), {strMode => '0700'});
testPathRemove($oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath());
storageTest()->pathCreate($oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath(), {strMode => '0700'});
testPathRemove($oHostDbPrimary->tablespacePath(1));
storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700'});
# Now the restore should work
$oHostDbPrimary->restore(
undef, 'latest',
{strOptionalParam => ' --db-include=test2 --db-include=test3 --buffer-size=16384', iRepo => $iRepoTotal});
# Test that the first database has not been restored since --db-include did not include test1
my ($strSHA1, $lSize) = storageTest()->hashSize($strDb1TablePath);
# Create a zeroed sparse file in the test directory that is the same size as the filenode.map. We need to use the posix
# driver directly to do this because handles cannot be passed back from the C code.
my $oStorageTrunc = new pgBackRestTest::Common::Storage($self->testPath(), new pgBackRestTest::Common::StoragePosix());
my $strTestTable = $self->testPath() . "/testtable";
my $oDestinationFileIo = $oStorageTrunc->openWrite($strTestTable);
$oDestinationFileIo->open();
# Truncate to the original size which will create a sparse file.
if (!truncate($oDestinationFileIo->handle(), $lSize))
{
confess "unable to truncate '$strTestTable' with handle " . $oDestinationFileIo->handle();
}
$oDestinationFileIo->close();
# Confirm the test filenode.map and the database test1 filenode.map are zeroed
my ($strSHA1Test, $lSizeTest) = storageTest()->hashSize($strTestTable);
$self->testResult(sub {($strSHA1Test eq $strSHA1) && ($lSizeTest == $lSize) && ($strSHA1 ne $strDb1TableSha1)},
true, 'database test1 not restored');
$oHostDbPrimary->clusterStart();
$oHostDbPrimary->sqlSelectOneTest('select message from test', $strNameMessage);
# Once the cluster is back online, make sure the database & table in the tablespace exists properly
$oHostDbPrimary->sqlSelectOneTest('select id from test_ts1', 2, {strDb => 'test2'});
$oHostDbPrimary->sqlDisconnect({strDb => 'test2'});
$oHostDbPrimary->sqlSelectOneTest('select id from test3_exists', 1, {strDb => 'test3'});
$oHostDbPrimary->sqlDisconnect({strDb => 'test3'});
# The tablespace path should exist and have files in it
my $strTablespacePath = $oHostDbPrimary->tablespacePath(1);
# Backup info will have the catalog number
my $oBackupInfo = new pgBackRestDoc::Common::Ini(
storageRepo(), $oHostBackup->repoBackupPath(FILE_BACKUP_INFO),
{bLoad => false, strContent => ${storageRepo()->get($oHostBackup->repoBackupPath(FILE_BACKUP_INFO))}});
# Construct the special path
$strTablespacePath .=
'/PG_' . $oHostDbPrimary->pgVersion() . qw{_} . $oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG);
# Check that path exists
if (!storageTest()->pathExists($strTablespacePath))
{
confess &log(ASSERT, "unable to find tablespace path '${strTablespacePath}'");
}
# Make sure there are some files in the tablespace path
if (grep(!/^PG\_VERSION$/i, storageTest()->list($strTablespacePath)) == 0)
{
confess &log(ASSERT, "no files found in tablespace path '${strTablespacePath}'");
}
# This table should exist to prove that the tablespace was restored. It has not been updated since it was created so it
# should not be created by any full page writes. Once it is verified to exist it can be dropped.
$oHostDbPrimary->sqlSelectOneTest("select count(*) from test_exists", 0);
$oHostDbPrimary->sqlExecute('drop table test_exists');
# Now it should be OK to drop database test2 and test3
$oHostDbPrimary->sqlExecute('drop database test2', {bAutoCommit => true});
# The test table lives in ts1 so it needs to be moved or dropped
$oHostDbPrimary->sqlExecute('alter table test set tablespace pg_default');
# And drop the tablespace
$oHostDbPrimary->sqlExecute('drop database test3', {bAutoCommit => true});
$oHostDbPrimary->sqlExecute("drop tablespace ts1", {bAutoCommit => true});
# Restore (restore type = immediate, inclusive)
#---------------------------------------------------------------------------------------------------------------------------
if ($oHostDbPrimary->pgVersion() >= PG_VERSION_94)
{
&log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_IMMEDIATE);
$oHostDbPrimary->clusterStop();
$oHostDbPrimary->restore(
undef, $strFullBackup, {bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_IMMEDIATE, strTargetAction => 'promote'});
$oHostDbPrimary->clusterStart();
$oHostDbPrimary->sqlSelectOneTest('select message from test', ($strStandbyMessage));
}
# Restore (restore type = xid, inclusive)
#---------------------------------------------------------------------------------------------------------------------------
my $strRecoveryFile = undef;
&log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_XID);
$oHostDbPrimary->clusterStop();
executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath() . "/*");
executeTest('rm -rf ' . $oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath() . '/*');
$oHostDbPrimary->restore(
undef, $strIncrBackup,
{bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_XID, strTarget => $strXidTarget, strTargetAction => 'promote',
strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef,
strOptionalParam => '--tablespace-map-all=../../tablespace', bTablespace => false,
iRepo => $iRepoTotal});
# Save recovery file to test so we can use it in the next test
$strRecoveryFile = $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'postgresql.auto.conf' : DB_FILE_RECOVERYCONF;
storageTest()->copy(
$oHostDbPrimary->dbBasePath() . qw{/} . $strRecoveryFile, $self->testPath() . qw{/} . $strRecoveryFile);
$oHostDbPrimary->clusterStart();
$oHostDbPrimary->sqlSelectOneTest('select message from test', $strXidMessage);
$oHostDbPrimary->sqlExecute("update test set message = '$strTimelineMessage'");
# Restore (restore type = preserve, inclusive)
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_PRESERVE);
$oHostDbPrimary->clusterStop();
executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath() . "/*");
executeTest('rm -rf ' . $oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath() . '/*');
executeTest('rm -rf ' . $oHostDbPrimary->tablespacePath(1) . "/*");
# Restore recovery file that was saved in last test
storageTest()->move($self->testPath . "/${strRecoveryFile}", $oHostDbPrimary->dbBasePath() . "/${strRecoveryFile}");
# Also touch recovery.signal when required
if ($oHostDbPrimary->pgVersion() >= PG_VERSION_12)
{
storageTest()->put($oHostDbPrimary->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL);
}
$oHostDbPrimary->restore(undef, 'latest', {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE});
$oHostDbPrimary->clusterStart();
$oHostDbPrimary->sqlSelectOneTest('select message from test', $strXidMessage);
$oHostDbPrimary->sqlExecute("update test set message = '$strTimelineMessage'");
# Restore (restore type = time, inclusive, automatically select backup) - there is no exclusive time test because I can't
# find a way to find the exact commit time of a transaction.
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_TIME);
$oHostDbPrimary->clusterStop();
$oHostDbPrimary->restore(
undef, 'latest',
{bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_TIME, strTarget => $strTimeTarget, strTargetAction => 'promote',
strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef,
strBackupExpected => $strFullBackup});
$oHostDbPrimary->clusterStart();
$oHostDbPrimary->sqlSelectOneTest('select message from test', $strTimeMessage);
# Restore (restore type = xid, exclusive)
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_XID);
$oHostDbPrimary->clusterStop();
$oHostDbPrimary->restore(
undef, $strIncrBackup,
{bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_XID, strTarget => $strXidTarget, bTargetExclusive => true,
strTargetAction => 'promote',
strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef,
iRepo => $iRepoTotal});
$oHostDbPrimary->clusterStart();
$oHostDbPrimary->sqlSelectOneTest('select message from test', $strIncrMessage);
# Restore (restore type = name)
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_NAME);
$oHostDbPrimary->clusterStop();
$oHostDbPrimary->restore(
undef, 'latest',
{bDelta => true, bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_NAME, strTarget => $strNameTarget,
strTargetAction => 'promote',
strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef});
$oHostDbPrimary->clusterStart();
$oHostDbPrimary->sqlSelectOneTest('select message from test', $strNameMessage);
# Restore (restore type = default, timeline = created by type = xid, inclusive recovery)
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_DEFAULT);
$oHostDbPrimary->clusterStop();
# The timeline to use for this test is subject to change based on tests being added or removed above. The best thing would
# be to automatically grab the timeline after the restore, but since this test has been stable for a long time it does not
# seem worth the effort to automate.
$oHostDbPrimary->restore(
undef, $strIncrBackup,
{bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, strTargetTimeline => 4, iRepo => $iRepoTotal});
$oHostDbPrimary->clusterStart({bHotStandby => true});
$oHostDbPrimary->sqlSelectOneTest('select message from test', $strTimelineMessage, {iTimeout => 120});
# Stop clusters to catch any errors in the postgres log
#---------------------------------------------------------------------------------------------------------------------------
$oHostDbPrimary->clusterStop();
# Stanza-delete --force without access to pgbackrest on database host. This test is not version specific so is run on only
# one version.
#---------------------------------------------------------------------------------------------------------------------------
if ($bNonVersionSpecific)
{
# Make sure this test has a backup host to work with
confess "test must run with backup dst = " . HOST_BACKUP if !$bHostBackup;
$oHostDbPrimary->stop();
$oHostBackup->stop({strStanza => $self->stanza});
$oHostBackup->stanzaDelete(
"delete stanza with --force when pgbackrest on pg host not accessible", {strOptionalParam => ' --force'});
$oHostDbPrimary->start();
$oHostBackup->start();
}
}
}
1;

6
test/src/build/config/config.yaml
View File

@ -70,6 +70,12 @@ option:
command:
test: {}
pg-version:
type: string
default: invalid
command:
test: {}
profile:
type: boolean
default: false

10
test/src/build/help/help.xml
View File

@ -120,6 +120,16 @@
<example>n</example>
</option>
<option id="pg-version">
<summary><postgres/> version for integration test.</summary>
<text>
<p>Version of <postgres/> to use for the integration test.</p>
</text>
<example>16</example>
</option>
<option id="profile">
<summary>Generate profile report.</summary>

32
test/src/command/test/build.c
View File

@ -29,15 +29,18 @@ Constants
/**********************************************************************************************************************************/
TestBuild *
testBldNew(
const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId,
const TestDefModule *const module, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime,
const String *const timeZone, const bool coverage, const bool profile, const bool optimize, const bool backTrace)
const String *const pathRepo, const String *const pathTest, const String *const vm, const String *const vmInt,
const unsigned int vmId, const String *const pgVersion, const TestDefModule *const module, const unsigned int test,
const uint64_t scale, const LogLevel logLevel, const bool logTime, const String *const timeZone, const bool coverage,
const bool profile, const bool optimize, const bool backTrace)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, pathRepo);
FUNCTION_LOG_PARAM(STRING, pathTest);
FUNCTION_LOG_PARAM(STRING, vm);
FUNCTION_LOG_PARAM(STRING, vmInt);
FUNCTION_LOG_PARAM(UINT, vmId);
FUNCTION_LOG_PARAM(STRING, pgVersion);
FUNCTION_LOG_PARAM_P(VOID, module);
FUNCTION_LOG_PARAM(UINT, test);
FUNCTION_LOG_PARAM(UINT64, scale);
@ -53,6 +56,7 @@ testBldNew(
ASSERT(pathRepo != NULL);
ASSERT(pathTest != NULL);
ASSERT(vm != NULL);
ASSERT(vmInt != NULL);
ASSERT(module != NULL);
ASSERT(scale != 0);
@ -65,7 +69,9 @@ testBldNew(
.pathRepo = strDup(pathRepo),
.pathTest = strDup(pathTest),
.vm = strDup(vm),
.vmInt = strDup(vmInt),
.vmId = vmId,
.pgVersion = strDup(pgVersion),
.module = module,
.test = test,
.scale = scale,
@ -315,6 +321,11 @@ testBldUnit(TestBuild *const this)
for (unsigned int shimIdx = 0; shimIdx < lstSize(module->shimList); shimIdx++)
{
const TestDefShim *const shim = lstGet(module->shimList, shimIdx);
// Skip this shim for integration tests
if (module->type == testDefTypeIntegration && !shim->integration)
continue;
const String *const shimFile = strNewFmt("%s.c", strZ(cmdBldPathModule(shim->name)));
String *const shimC = strCatBuf(
@ -332,6 +343,11 @@ testBldUnit(TestBuild *const this)
for (unsigned int harnessIdx = 0; harnessIdx < lstSize(module->harnessList); harnessIdx++)
{
const TestDefHarness *const harness = lstGet(module->harnessList, harnessIdx);
// Skip this harness for integration tests
if (module->type == testDefTypeIntegration && !harness->integration)
continue;
const String *const harnessFile = strNewFmt("test/src/common/%s.c", strZ(bldEnum("harness", harness->name)));
const String *harnessPath = strNewFmt("%s/%s", strZ(pathRepo), strZ(harnessFile));
@ -634,8 +650,8 @@ testBldUnit(TestBuild *const this)
const String *const pathProjectExe = storagePathP(
testBldStorageTest(this),
strNewFmt(
"%s/%s%s/" PROJECT_BIN, strEqZ(testBldVm(this), "none") ? "build" : "bin", strZ(testBldVm(this)),
strEqZ(testBldVm(this), "none") ? "/src" : ""));
"%s/%s%s/" PROJECT_BIN, strEqZ(testBldVmInt(this), "none") ? "build" : "bin", strZ(testBldVmInt(this)),
strEqZ(testBldVmInt(this), "none") ? "/src" : ""));
strReplace(testC, STRDEF("{[C_TEST_PROJECT_EXE]}"), pathProjectExe);
// Path to source -- used to construct __FILENAME__ tests
@ -671,6 +687,12 @@ testBldUnit(TestBuild *const this)
strReplace(testC, STRDEF("{[C_TEST_USER_LEN]}"), strNewFmt("%zu", strSize(userName())));
strReplace(testC, STRDEF("{[C_TEST_USER_ID]}"), strNewFmt("%u", userId()));
// VM for integration testing
strReplace(testC, STRDEF("{[C_TEST_VM]}"), testBldVmInt(this));
// PostgreSQL version for integration testing
strReplace(testC, STRDEF("{[C_TEST_PG_VERSION]}"), testBldPgVersion(this));
// Test id
strReplace(testC, STRDEF("{[C_TEST_IDX]}"), strNewFmt("%u", testBldVmId(this)));

22
test/src/command/test/build.h
View File

@ -19,9 +19,9 @@ typedef struct TestBuild TestBuild;
Constructors
***********************************************************************************************************************************/
TestBuild *testBldNew(
const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const TestDefModule *module,
unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, bool coverage, bool profile,
bool optimize, bool backTrace);
const String *pathRepo, const String *pathTest, const String *const vm, const String *const vmInt, unsigned int vmId,
const String *pgVersion, const TestDefModule *module, unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime,
const String *timeZone, bool coverage, bool profile, bool optimize, bool backTrace);
/***********************************************************************************************************************************
Getters/Setters
@ -33,7 +33,9 @@ typedef struct TestBuildPub
const Storage *storageRepo; // Repository storage
const Storage *storageTest; // Test storage
const String *vm; // Vm to run the test on
const String *vmInt; // Vm to run the test on for integration
unsigned int vmId; // Vm id (0-based) to run the test on
const String *pgVersion; // Pg version to run the test on
const TestDefModule *module; // Module definition
unsigned int test; // Specific test to run (0 if all)
LogLevel logLevel; // Test log level
@ -82,6 +84,13 @@ testBldVm(const TestBuild *const this)
return THIS_PUB(TestBuild)->vm;
}
// Vm integration
FN_INLINE_ALWAYS const String *
testBldVmInt(const TestBuild *const this)
{
return THIS_PUB(TestBuild)->vmInt;
}
// Vm id
FN_INLINE_ALWAYS unsigned int
testBldVmId(const TestBuild *const this)
@ -89,6 +98,13 @@ testBldVmId(const TestBuild *const this)
return THIS_PUB(TestBuild)->vmId;
}
// Pg version
FN_INLINE_ALWAYS const String *
testBldPgVersion(const TestBuild *const this)
{
return THIS_PUB(TestBuild)->pgVersion;
}
// Test Definition
FN_INLINE_ALWAYS const TestDefModule *
testBldModule(const TestBuild *const this)

89
test/src/command/test/define.c
View File

@ -137,7 +137,7 @@ testDefParseModuleList(Yaml *const yaml, List *const moduleList)
}
else if (strEqZ(subModuleDef.value, "harness"))
{
TestDefHarness testDefHarness = {0};
TestDefHarness testDefHarness = {.integration = true};
StringList *harnessIncludeList = strLstNew();
if (yamlEventPeek(yaml).type == yamlEventTypeScalar)
@ -148,44 +148,59 @@ testDefParseModuleList(Yaml *const yaml, List *const moduleList)
{
YAML_MAP_BEGIN(yaml)
{
yamlScalarNextCheckZ(yaml, "name");
testDefHarness.name = yamlScalarNext(yaml).value;
const String *const type = yamlScalarNext(yaml).value;
yamlScalarNextCheckZ(yaml, "shim");
YAML_MAP_BEGIN(yaml)
if (strEqZ(type, "name"))
{
const String *const shim = yamlScalarNext(yaml).value;
strLstAdd(harnessIncludeList, shim);
if (yamlEventPeek(yaml).type == yamlEventTypeScalar)
{
yamlScalarNext(yaml);
}
else
{
TestDefShim testDefShim = {.name = shim, .functionList = strLstNew()};
YAML_MAP_BEGIN(yaml)
{
yamlScalarNextCheckZ(yaml, "function");
StringList *const functionList = strLstNew();
YAML_SEQ_BEGIN(yaml)
{
strLstAdd(functionList, yamlScalarNext(yaml).value);
}
YAML_SEQ_END();
testDefShim.functionList = functionList;
}
YAML_MAP_END();
lstAdd(globalShimList, &testDefShim);
}
testDefHarness.name = yamlScalarNext(yaml).value;
}
else if (strEqZ(type, "integration"))
{
testDefHarness.integration = yamlBoolParse(yamlScalarNext(yaml));
}
else
{
CHECK_FMT(AssertError, strEqZ(type, "shim"), "invalid key '%s'", strZ(type));
YAML_MAP_BEGIN(yaml)
{
const String *const shim = yamlScalarNext(yaml).value;
strLstAdd(harnessIncludeList, shim);
if (yamlEventPeek(yaml).type == yamlEventTypeScalar)
{
yamlScalarNext(yaml);
}
else
{
TestDefShim testDefShim =
{
.name = shim,
.integration = testDefHarness.integration,
.functionList = strLstNew()
};
YAML_MAP_BEGIN(yaml)
{
yamlScalarNextCheckZ(yaml, "function");
StringList *const functionList = strLstNew();
YAML_SEQ_BEGIN(yaml)
{
strLstAdd(functionList, yamlScalarNext(yaml).value);
}
YAML_SEQ_END();
testDefShim.functionList = functionList;
}
YAML_MAP_END();
lstAdd(globalShimList, &testDefShim);
}
}
YAML_MAP_END();
}
YAML_MAP_END();
}
YAML_MAP_END();
}
@ -256,6 +271,7 @@ testDefParseModuleList(Yaml *const yaml, List *const moduleList)
const TestDefHarness testDefHarness =
{
.name = strDup(globalHarness->name),
.integration = globalHarness->integration,
.includeList = strLstDup(globalHarness->includeList),
};
@ -277,6 +293,7 @@ testDefParseModuleList(Yaml *const yaml, List *const moduleList)
const TestDefShim testDefShim =
{
.name = strDup(globalShim->name),
.integration = globalShim->integration,
.functionList = strLstDup(globalShim->functionList),
};

2
test/src/command/test/define.h
View File

@ -28,6 +28,7 @@ typedef struct TestDefCoverage
typedef struct TestDefHarness
{
const String *name; // Harness module name
bool integration; // Include in integration tests?
const StringList *includeList; // List of modules to include directly in harness
} TestDefHarness;
@ -35,6 +36,7 @@ typedef struct TestDefHarness
typedef struct TestDefShim
{
const String *name; // Shim module name
bool integration; // Include in integration tests?
const StringList *functionList; // List of functions to shim
} TestDefShim;

19
test/src/command/test/test.c
View File

@ -49,15 +49,17 @@ cmdTestPathCreate(const Storage *const storage, const String *const path)
/**********************************************************************************************************************************/
void
cmdTest(
const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId,
const String *moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime,
const String *const timeZone, const bool coverage, const bool profile, const bool optimize, const bool backTrace)
const String *const pathRepo, const String *const pathTest, const String *vm, const unsigned int vmId,
const String *const pgVersion, const String *moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel,
const bool logTime, const String *const timeZone, const bool coverage, const bool profile, const bool optimize,
const bool backTrace)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, pathRepo);
FUNCTION_LOG_PARAM(STRING, pathTest);
FUNCTION_LOG_PARAM(STRING, vm);
FUNCTION_LOG_PARAM(UINT, vmId);
FUNCTION_LOG_PARAM(STRING, pgVersion);
FUNCTION_LOG_PARAM(STRING, moduleName);
FUNCTION_LOG_PARAM(UINT, test);
FUNCTION_LOG_PARAM(UINT64, scale);
@ -80,6 +82,13 @@ cmdTest(
CHECK_FMT(ParamInvalidError, module != NULL, "'%s' is not a valid test", strZ(moduleName));
// Vm used for integration and the pgbackrest binary
const String *const vmInt = vm;
// If integration then the vm should be none. Integration tests do not run in containers but instead spawn their own.
if (module->type == testDefTypeIntegration)
vm = strNewZ("none");
// Build test
bool buildRetry = false;
const String *const pathUnit = strNewFmt("%s/unit-%u/%s", strZ(pathTest), vmId, strZ(vm));
@ -92,8 +101,8 @@ cmdTest(
{
// Build unit
TestBuild *const testBld = testBldNew(
pathRepo, pathTest, vm, vmId, module, test, scale, logLevel, logTime, timeZone, coverage, profile, optimize,
backTrace);
pathRepo, pathTest, vm, vmInt, vmId, pgVersion, module, test, scale, logLevel, logTime, timeZone, coverage,
profile, optimize, backTrace);
testBldUnit(testBld);
// Meson setup

6
test/src/command/test/test.h
View File

@ -13,8 +13,8 @@ Perform a test.
Functions
***********************************************************************************************************************************/
void cmdTest(
const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *moduleName,
unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, bool coverage, bool profile,
bool optimize, bool backTrace);
const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *pgVersion,
const String *moduleName, unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone,
bool coverage, bool profile, bool optimize, bool backTrace);
#endif

1301
test/src/common/harnessHost.c Normal file
View File

File diff suppressed because it is too large Load Diff

490
test/src/common/harnessHost.h Normal file
View File

@ -0,0 +1,490 @@
/***********************************************************************************************************************************
Host Harness
***********************************************************************************************************************************/
#ifndef TEST_COMMON_HARNESS_HOST_H
#define TEST_COMMON_HARNESS_HOST_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct HrnHost HrnHost;
#include "common/type/object.h"
#include "common/type/string.h"
#include "postgres/client.h"
#include "common/harnessTest.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
// Host constants
#define HRN_HOST_AZURE STORAGE_AZURE_TYPE
#define HRN_HOST_AZURE_Z "azure"
#define HRN_HOST_GCS STORAGE_GCS_TYPE
#define HRN_HOST_GCS_Z "gcs"
#define HRN_HOST_POSIX STORAGE_POSIX_TYPE
#define HRN_HOST_POSIX_Z "posix"
#define HRN_HOST_PG1 STRID6("pg1", 0x1d1d01)
#define HRN_HOST_PG1_Z "pg1"
#define HRN_HOST_PG2 STRID5("pg2", 0x70f00)
#define HRN_HOST_PG2_Z "pg2"
#define HRN_HOST_REPO STRID5("repo", 0x7c0b20)
#define HRN_HOST_REPO_Z "repo"
#define HRN_HOST_S3 STORAGE_S3_TYPE
#define HRN_HOST_S3_Z "s3"
#define HRN_HOST_SFTP STORAGE_SFTP_TYPE
#define HRN_HOST_SFTP_Z "sftp"
// Test stanza
#define HRN_STANZA "test"
/***********************************************************************************************************************************
Test
***********************************************************************************************************************************/
typedef struct HrnHostTestDefine
{
const char *pg; // PostgreSQL version
const char *repo; // Host acting as repository
bool tls; // Use TLS instead of SSH for remote protocol?
const char *stg; // Storage type
bool enc; // Encryption
const char *cmp; // Compression type
unsigned int rt; // Repository total
bool bnd; // Bundling enabled?
bool bi; // Block incremental enabled?
} HrnHostTestDefine;
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
typedef struct HrnHostNewParam
{
VAR_PARAM_HEADER;
const String *entryPoint; // Container entry point
const String *user; // Container user
const String *option; // Options (e.g. -v)
const String *param; // Parameters (passed to the container command)
bool noUpdateHosts; // Do not update /etc/hosts file on this host
const String *dataPath; // Path where container data is stored
bool pgStandby; // Pg standby?
bool isPg; // Is a pg host?
bool isRepo; // Is a repo host?
} HrnHostNewParam;
#define hrnHostNewP(name, container, image, ...) \
hrnHostNew(name, container, image, (HrnHostNewParam){VAR_PARAM_INIT, __VA_ARGS__})
HrnHost *hrnHostNew(StringId id, const String *container, const String *image, HrnHostNewParam param);
/***********************************************************************************************************************************
Getters/Setters
***********************************************************************************************************************************/
typedef struct HrnHostPub
{
StringId id; // Host id
const String *name; // Host name
const String *container; // Container name
const String *image; // Container image
const String *ip; // IP address
const String *user; // Container user
const String *dataPath; // Data path
const Storage *dataStorage; // Data storage
const String *logPath; // Log path
const String *brBin; // pgBackRest binary
bool pgStandby; // Pg Standby
const String *pgPath; // Pg path
const String *pgDataPath; // Pg data path
const String *pgTsPath; // Pg tablespace path
const String *pgLogFile; // Pg log path
const Storage *pgStorage; // Pg storage
const String *repo1Path; // Repo1 path
Storage *repo1Storage; // Repo1 storage
const String *repo2Path; // Repo2 path
Storage *repo2Storage; // Repo2 storage
const String *spoolPath; // Spool path
bool isPg; // Is a pg host?
bool isRepo; // Is a repo host?
bool updateHosts; // Is /etc/hosts file updated on this host?
} HrnHostPub;
// Container name
FN_INLINE_ALWAYS const String *
hrnHostContainer(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->container;
}
// Host data path
FN_INLINE_ALWAYS const String *
hrnHostDataPath(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->dataPath;
}
// Host data storage
FN_INLINE_ALWAYS const Storage *
hrnHostDataStorage(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->dataStorage;
}
// Container image
FN_INLINE_ALWAYS const String *
hrnHostImage(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->image;
}
// Host id
FN_INLINE_ALWAYS StringId
hrnHostId(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->id;
}
// Container IP address
FN_INLINE_ALWAYS const String *
hrnHostIp(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->ip;
}
// Host name
FN_INLINE_ALWAYS const String *
hrnHostName(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->name;
}
// Is a pg host?
FN_INLINE_ALWAYS bool
hrnHostIsPg(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->isPg;
}
// Is a repo host?
FN_INLINE_ALWAYS bool
hrnHostIsRepo(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->isRepo;
}
// Log path
FN_INLINE_ALWAYS const String *
hrnHostLogPath(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->logPath;
}
// pgBackRest binary
FN_INLINE_ALWAYS const String *
hrnHostBrBin(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->brBin;
}
// Pg bin path
const String *hrnHostPgBinPath(HrnHost *this);
// Pg client
PgClient *hrnHostPgClient(HrnHost *this);
// Pg path
FN_INLINE_ALWAYS const String *
hrnHostPgPath(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->pgPath;
}
// Pg log file
FN_INLINE_ALWAYS const String *
hrnHostPgLogFile(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->pgLogFile;
}
// Pg data path
FN_INLINE_ALWAYS const String *
hrnHostPgDataPath(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->pgDataPath;
}
// Pg standby?
FN_INLINE_ALWAYS bool
hrnHostPgStandby(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->pgStandby;
}
// Host data storage
FN_INLINE_ALWAYS const Storage *
hrnHostPgStorage(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->pgStorage;
}
// Pg tablespace path
FN_INLINE_ALWAYS const String *
hrnHostPgTsPath(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->pgTsPath;
}
// Repo1 path
FN_INLINE_ALWAYS const String *
hrnHostRepo1Path(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->repo1Path;
}
// Repo1 storage
FN_INLINE_ALWAYS const Storage *
hrnHostRepo1Storage(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->repo1Storage;
}
// Repo2 path
FN_INLINE_ALWAYS const String *
hrnHostRepo2Path(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->repo2Path;
}
// Repo2 storage
FN_INLINE_ALWAYS const Storage *
hrnHostRepo2Storage(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->repo2Storage;
}
// Spool path
FN_INLINE_ALWAYS const String *
hrnHostSpoolPath(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->spoolPath;
}
// Host name
FN_INLINE_ALWAYS bool
hrnHostUpdateHosts(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->updateHosts;
}
// Container user
FN_INLINE_ALWAYS const String *
hrnHostUser(const HrnHost *const this)
{
return THIS_PUB(HrnHost)->user;
}
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Execute a command
typedef struct HrnHostExecParam
{
VAR_PARAM_HEADER;
const String *user; // User to execute command
int resultExpect; // Expected result, if not 0
} HrnHostExecParam;
#define hrnHostExecP(this, command, ...) \
hrnHostExec(this, command, (HrnHostExecParam){VAR_PARAM_INIT, __VA_ARGS__})
String *hrnHostExec(HrnHost *this, const String *command, HrnHostExecParam param);
// Execute pgbackrest
#define TEST_HOST_BR(this, command, ...) \
do \
{ \
const HrnHostExecBrParam param = {VAR_PARAM_INIT, __VA_ARGS__}; \
\
TEST_RESULT_INFO_FMT( \
"%s: pgbackrest%s %s%s%s", strZ(hrnHostName(this)), param.option == NULL ? "" : zNewFmt(" %s", param.option), command, \
param.param == NULL ? "" : zNewFmt(" %s", param.param), \
param.resultExpect != 0 ? zNewFmt(" -- result %d", param.resultExpect) : ""); \
hrnHostExecBrP(this, command , __VA_ARGS__); \
} \
while (0);
typedef struct HrnHostExecBrParam
{
VAR_PARAM_HEADER;
const char *option; // Options
const char *param; // Parameters
int resultExpect; // Expected result, if not 0
} HrnHostExecBrParam;
#define hrnHostExecBrP(this, command, ...) \
hrnHostExecBr(this, command, (HrnHostExecBrParam){VAR_PARAM_INIT, __VA_ARGS__})
String *hrnHostExecBr(HrnHost *this, const char *command, HrnHostExecBrParam param);
// Create/start/stop Pg cluster
#define HRN_HOST_PG_CREATE(this) \
do \
{ \
TEST_RESULT_INFO_FMT("%s: create pg cluster", strZ(hrnHostName(this))); \
hrnHostPgCreate(this); \
} \
while (0)
void hrnHostPgCreate(HrnHost *this);
#define HRN_HOST_PG_START(this) \
do \
{ \
TEST_RESULT_INFO_FMT("%s: start pg cluster", strZ(hrnHostName(this))); \
hrnHostPgStart(this); \
} \
while (0)
void hrnHostPgStart(HrnHost *this);
#define HRN_HOST_PG_STOP(this) \
do \
{ \
TEST_RESULT_INFO_FMT("%s: stop pg cluster", strZ(hrnHostName(this))); \
hrnHostPgStop(this); \
} \
while (0)
void hrnHostPgStop(HrnHost *this);
// Query
PackRead *hrnHostSql(HrnHost *this, const String *statement, const PgClientQueryResult resultType);
FN_INLINE_ALWAYS PackRead *
hrnHostSqlLog(HrnHost *const this, int line, const char *const statement, const PgClientQueryResult resultType)
{
TEST_RESULT_INFO_LINE_FMT(line, "%s: %s", strZ(hrnHostName(this)), statement);
return hrnHostSql(this, STR(statement), resultType);
}
#define HRN_HOST_SQL(this, statement, resultType) \
hrnHostSqlLog(this, __LINE__, statement, resultType)
// Exec statement
void hrnHostSqlExec(HrnHost *this, const String *statement);
#define HRN_HOST_SQL_EXEC(this, statement) \
do \
{ \
TEST_RESULT_INFO_FMT("%s: %s", strZ(hrnHostName(this)), statement); \
hrnHostSqlExec(this, STR(statement)); \
} while (0)
// Query a single value
FN_INLINE_ALWAYS PackRead *
hrnHostSqlValue(HrnHost *const this, const char *const statement)
{
return hrnHostSql(this, STR(statement), pgClientQueryResultColumn);
}
#define HRN_HOST_SQL_VALUE(this, statement) \
HRN_HOST_SQL(this, statement, pgClientQueryResulColumn)
// Test a single value
void hrnHostSqlTest(HrnHost *this, const String *statement, const String *expected);
#define TEST_HOST_SQL_ONE_STR_Z(this, statement, expected) \
do \
{ \
TEST_RESULT_INFO_FMT("%s: %s == '%s'", strZ(hrnHostName(this)), statement, expected); \
hrnTestResultBegin(#statement, true); \
hrnHostSqlTest(this, STR(statement), STR(expected)); \
} \
while (0)
// Begin transaction
FN_INLINE_ALWAYS void
hrnHostSqlBegin(HrnHost *const this)
{
hrnHostSqlExec(this, STRDEF("begin"));
}
// Commit transaction
FN_INLINE_ALWAYS void
hrnHostSqlCommit(HrnHost *const this)
{
hrnHostSqlExec(this, STRDEF("commit"));
}
// Switch WAL segment
#define HRN_HOST_WAL_SWITCH(this) \
HRN_HOST_SQL_EXEC(this, zNewFmt("select pg_switch_%s()::text", strZ(pgWalName(hrnHostPgVersion()))))
/***********************************************************************************************************************************
Helper functions
***********************************************************************************************************************************/
// Cipher Type
CipherType hrnHostCipherType(void);
// Cipher Passphrase
const String *hrnHostCipherPass(void);
// Compress Type
CompressType hrnHostCompressType(void);
// Non version-specific testing enabled
bool hrnHostNonVersionSpecific(void);
// Pg version
unsigned int hrnHostPgVersion(void);
// Pg1 host
HrnHost *hrnHostPg1(void);
// Pg2 host
HrnHost *hrnHostPg2(void);
// Repo host
HrnHost *hrnHostRepo(void);
// Repo total
unsigned int hrnHostRepoTotal(void);
// Get a host by name
HrnHost *hrnHostGet(StringId id);
// Run hosts and build configuration based on test matrix
void hrnHostBuild(int line, const HrnHostTestDefine *testMatrix, size_t testMatrixSize);
#define HRN_HOST_BUILD(testMatrix) \
hrnHostBuild(__LINE__, testMatrix, LENGTH_OF(testMatrix))
// Update configuration
typedef struct HrnHostConfigUpdateParam
{
VAR_PARAM_HEADER;
const Variant *archiveAsync; // Update async archiving?
} HrnHostConfigUpdateParam;
#define hrnHostConfigUpdateP(...) \
hrnHostConfigUpdate((HrnHostConfigUpdateParam){VAR_PARAM_INIT, __VA_ARGS__})
void hrnHostConfigUpdate(const HrnHostConfigUpdateParam param);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
FN_INLINE_ALWAYS void
hrnHostFree(HrnHost *const this)
{
objFree(this);
}
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_LOG_HRN_HOST_TYPE \
HrnHost *
#define FUNCTION_LOG_HRN_HOST_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "HrnHost", buffer, bufferSize)
#endif

35
test/src/common/harnessTest.c
View File

@ -37,6 +37,9 @@ static bool testLogExpectData = false;
static unsigned int testIdxData = 0;
static bool testTiming = true;
static const char *testPathData = NULL;
static const char *testUserData = NULL;
static const char *testVmData = NULL;
static const char *testPgVersionData = NULL;
static const char *testDataPathData = NULL;
static const char *testRepoPathData = NULL;
@ -67,8 +70,9 @@ Initialize harness
***********************************************************************************************************************************/
void
hrnInit(
const char *testExe, const char *testProjectExe, bool testContainer, bool testLogExpect, unsigned int testIdx, bool timing,
const char *testPath, const char *testDataPath, const char *testRepoPath)
const char *const testExe, const char *const testProjectExe, const bool testContainer, const bool testLogExpect,
const unsigned int testIdx, const bool timing, const char *const testPath, const char *const testUser, const char *const testVm,
const char *const testPgVersion,const char *const testDataPath, const char *const testRepoPath)
{
FUNCTION_HARNESS_VOID();
@ -81,6 +85,9 @@ hrnInit(
testIdxData = testIdx;
testTiming = timing;
testPathData = testPath;
testUserData = testUser;
testVmData = testVm;
testPgVersionData = testPgVersion;
testDataPathData = testDataPath;
testRepoPathData = testRepoPath;
@ -727,6 +734,30 @@ testPath(void)
FUNCTION_HARNESS_RETURN(STRINGZ, testPathData);
}
/**********************************************************************************************************************************/
const char *
testPgVersion(void)
{
FUNCTION_HARNESS_VOID();
FUNCTION_HARNESS_RETURN(STRINGZ, testPgVersionData);
}
/**********************************************************************************************************************************/
const char *
testUser(void)
{
FUNCTION_HARNESS_VOID();
FUNCTION_HARNESS_RETURN(STRINGZ, testUserData);
}
/**********************************************************************************************************************************/
const char *
testVm(void)
{
FUNCTION_HARNESS_VOID();
FUNCTION_HARNESS_RETURN(STRINGZ, testVmData);
}
/**********************************************************************************************************************************/
const char *
hrnPath(void)

17
test/src/common/harnessTest.h
View File

@ -67,6 +67,15 @@ bool testContainer(void);
// parallel.
unsigned int testIdx(void);
// PostgreSQL version for integration testing
const char *testPgVersion(void);
// User running the test
const char *testUser(void);
// VM for integration testing
const char *testVm(void);
/***********************************************************************************************************************************
Test that an expected error is actually thrown and error when it isn't
***********************************************************************************************************************************/
@ -142,6 +151,14 @@ Output information about the test
puts(comment); \
fflush(stdout);
#define TEST_RESULT_INFO_LINE_FMT(line, comment, ...) \
hrnTestLogPrefix(line); \
printf(comment "\n", __VA_ARGS__); \
fflush(stdout)
#define TEST_RESULT_INFO_FMT(comment, ...) \
TEST_RESULT_INFO_LINE_FMT(__LINE__, comment, __VA_ARGS__)
/***********************************************************************************************************************************
Test that a void statement returns and does not throw an error
***********************************************************************************************************************************/

3
test/src/common/harnessTest.intern.h
View File

@ -41,7 +41,8 @@ Functions
***********************************************************************************************************************************/
void hrnInit(
const char *testExe, const char *testProjectExe, bool testContainer, bool testLogExpect, unsigned int testIdx, bool timing,
const char *testPath, const char *testDataPath, const char *testRepoPath);
const char *testPath, const char *testUser, const char *testVm, const char *testPgVersion, const char *testDataPath,
const char *testRepoPath);
void hrnAdd(int run, bool selected);
void hrnComplete(void);

2
test/src/main.c
View File

@ -71,7 +71,7 @@ main(int argListSize, const char *argList[])
{
cmdTest(
cfgOptionStr(cfgOptRepoPath), cfgOptionStr(cfgOptTestPath), cfgOptionStr(cfgOptVm),
cfgOptionUInt(cfgOptVmId), strLstGet(cfgCommandParam(), 0),
cfgOptionUInt(cfgOptVmId), cfgOptionStr(cfgOptPgVersion), strLstGet(cfgCommandParam(), 0),
cfgOptionTest(cfgOptTest) ? cfgOptionUInt(cfgOptTest) : 0, cfgOptionUInt64(cfgOptScale),
logLevelEnum(cfgOptionStrId(cfgOptLogLevelTest)), cfgOptionBool(cfgOptLogTimestamp),
cfgOptionStrNull(cfgOptTz), cfgOptionBool(cfgOptCoverage), cfgOptionBool(cfgOptProfile),

9
test/src/module/common/execTest.c
View File

@ -135,7 +135,7 @@ testRun(void)
TEST_RESULT_VOID(execOpen(exec), "open cat exec");
kill(exec->processId, SIGKILL);
TEST_ERROR(execProcess(exec), ExecuteError, "cat terminated unexpectedly on signal 9");
TEST_ERROR(execProcess(exec, (ExecOneParam){0}), ExecuteError, "cat terminated unexpectedly on signal 9");
TEST_RESULT_VOID(execFree(exec), "free exec");
// -------------------------------------------------------------------------------------------------------------------------
@ -144,6 +144,13 @@ testRun(void)
TEST_ERROR(
execOneP(STRDEF("cat missing.txt")), UnknownError,
"cat missing.txt terminated unexpectedly [1]: cat: missing.txt: No such file or directory");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("exec ignores error");
TEST_RESULT_STR_Z(
execOneP(STRDEF("cat missing.txt"), .resultExpect = 1), "cat: missing.txt: No such file or directory\n",
"ignore error");
}
FUNCTION_HARNESS_RETURN_VOID();

404
test/src/module/integration/allTest.c Normal file
View File

@ -0,0 +1,404 @@
/***********************************************************************************************************************************
Real Integration Test
***********************************************************************************************************************************/
#include "common/crypto/common.h"
#include "config/config.h"
#include "info/infoBackup.h"
#include "postgres/interface.h"
#include "postgres/version.h"
#include "common/harnessErrorRetry.h"
#include "common/harnessHost.h"
#include "common/harnessPostgres.h"
#include "common/harnessStorage.h"
/***********************************************************************************************************************************
Test definition
***********************************************************************************************************************************/
static HrnHostTestDefine testMatrix[] =
{
// {uncrustify_off - struct alignment}
{.pg = "9.4", .repo = "pg2", .tls = 0, .stg = "azure", .enc = 1, .cmp = "lz4", .rt = 1, .bnd = 1, .bi = 0},
{.pg = "9.5", .repo = "repo", .tls = 1, .stg = "s3", .enc = 0, .cmp = "bz2", .rt = 1, .bnd = 1, .bi = 1},
{.pg = "9.6", .repo = "repo", .tls = 0, .stg = "posix", .enc = 0, .cmp = "none", .rt = 2, .bnd = 1, .bi = 1},
{.pg = "10", .repo = "pg2", .tls = 0, .stg = "sftp", .enc = 1, .cmp = "gz", .rt = 1, .bnd = 1, .bi = 0},
{.pg = "11", .repo = "repo", .tls = 1, .stg = "gcs", .enc = 0, .cmp = "zst", .rt = 2, .bnd = 0, .bi = 0},
{.pg = "12", .repo = "repo", .tls = 0, .stg = "s3", .enc = 1, .cmp = "lz4", .rt = 1, .bnd = 1, .bi = 1},
{.pg = "13", .repo = "pg2", .tls = 1, .stg = "sftp", .enc = 0, .cmp = "zst", .rt = 1, .bnd = 1, .bi = 1},
{.pg = "14", .repo = "repo", .tls = 0, .stg = "gcs", .enc = 0, .cmp = "lz4", .rt = 1, .bnd = 1, .bi = 0},
{.pg = "15", .repo = "pg2", .tls = 0, .stg = "azure", .enc = 0, .cmp = "none", .rt = 2, .bnd = 1, .bi = 1},
{.pg = "16", .repo = "repo", .tls = 0, .stg = "posix", .enc = 0, .cmp = "none", .rt = 1, .bnd = 0, .bi = 0},
// {uncrustify_on}
};
/***********************************************************************************************************************************
Test statuses
***********************************************************************************************************************************/
#define TEST_STATUS_FULL "full"
#define TEST_STATUS_INCR "incr"
#define TEST_STATUS_NAME "name"
#define TEST_STATUS_STANDBY "standby"
#define TEST_STATUS_TIME "time"
#define TEST_STATUS_TIMELINE "timeline"
#define TEST_STATUS_XID "xid"
#define TEST_RESTORE_POINT "pgbackrest"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
static void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// Enable error retry detail. This works because we are not trying to check exact error messages.
hrnErrorRetryDetailEnable();
// *****************************************************************************************************************************
if (testBegin("integration"))
{
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("build hosts");
HRN_HOST_BUILD(testMatrix);
HrnHost *const pg1 = hrnHostPg1();
HrnHost *const pg2 = hrnHostPg2();
HrnHost *const repo = hrnHostRepo();
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("create pg cluster");
{
HRN_HOST_PG_CREATE(pg1);
TEST_HOST_BR(repo, CFGCMD_STANZA_CREATE);
// Create tablespace
HRN_STORAGE_PATH_CREATE(hrnHostDataStorage(pg1), strZ(hrnHostPgTsPath(pg1)), .mode = 0700);
HRN_HOST_SQL_EXEC(pg1, zNewFmt("create tablespace ts1 location '%s'", strZ(hrnHostPgTsPath(pg1))));
// Init status table
HRN_HOST_SQL_EXEC(pg1, "create table status (message text not null) tablespace ts1");
HRN_HOST_WAL_SWITCH(pg1);
HRN_HOST_SQL_EXEC(pg1, "insert into status values ('" TEST_STATUS_FULL "')");
}
// Get ts1 tablespace oid
const unsigned int ts1Oid = pckReadU32P(hrnHostSqlValue(pg1, "select oid from pg_tablespace where spcname = 'ts1'"));
TEST_LOG_FMT("ts1 tablespace oid = %u", ts1Oid);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("check hosts (skip pg2 for now)");
{
if (pg1 != repo)
TEST_HOST_BR(pg1, CFGCMD_CHECK);
if (pg2 != repo)
TEST_HOST_BR(repo, CFGCMD_CHECK);
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary full backup");
{
// Full backup to repo 1
TEST_HOST_BR(repo, CFGCMD_BACKUP, .option = "--type=full --buffer-size=16KiB");
// Full backup to repo 2
if (hrnHostRepoTotal() == 2)
TEST_HOST_BR(repo, CFGCMD_BACKUP, .option = "--type=full --repo=2");
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("standby restore");
{
TEST_HOST_BR(
pg2, CFGCMD_RESTORE, .option = zNewFmt("--type=standby --tablespace-map=ts1='%s'", strZ(hrnHostPgTsPath(pg2))));
HRN_HOST_PG_START(pg2);
// Check standby
TEST_HOST_BR(pg2, CFGCMD_CHECK);
// Check that backup recovered completely
TEST_HOST_SQL_ONE_STR_Z(pg2, "select message from status", TEST_STATUS_FULL);
// Update message for standby
HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_STANDBY "'");
// Check that standby is streaming from the primary
TEST_HOST_SQL_ONE_STR_Z(
pg1,
zNewFmt(
"select client_addr || '-' || state from pg_stat_replication where client_addr = '%s'",
strZ(hrnHostIp(pg2))),
zNewFmt("%s/32-streaming", strZ(hrnHostIp(pg2))));
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("standby full backup and expire");
{
// Check backups before the backup so we know how many will exist after
const InfoBackup *infoBackup = infoBackupLoadFile(
hrnHostRepo1Storage(repo), STRDEF("backup/" HRN_STANZA "/backup.info"), hrnHostCipherType(), hrnHostCipherPass());
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 1, "backup total = 1");
TEST_HOST_BR(repo, CFGCMD_BACKUP, .option = "--type=full --backup-standby --repo1-retention-full=1 --no-expire-auto");
// Expire was disabled so the backup total has increased
infoBackup = infoBackupLoadFile(
hrnHostRepo1Storage(repo), STRDEF("backup/" HRN_STANZA "/backup.info"), hrnHostCipherType(), hrnHostCipherPass());
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, "backup total = 2");
// Now force an expire
TEST_HOST_BR(repo, CFGCMD_EXPIRE, .option = "--repo1-retention-full=1");
// Backup has been expired
infoBackup = infoBackupLoadFile(
hrnHostRepo1Storage(repo), STRDEF("backup/" HRN_STANZA "/backup.info"), hrnHostCipherType(), hrnHostCipherPass());
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 1, "backup total = 1");
// Stop the standby since restores to primary will break it
HRN_HOST_PG_STOP(pg2);
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("exercise async archiving");
hrnHostConfigUpdateP(.archiveAsync = BOOL_TRUE_VAR);
HRN_HOST_SQL_EXEC(pg1, "create table wal_activity (id int)");
HRN_HOST_WAL_SWITCH(pg1);
HRN_HOST_SQL_EXEC(pg1, "insert into wal_activity values (1)");
HRN_HOST_WAL_SWITCH(pg1);
HRN_HOST_SQL_EXEC(pg1, "insert into wal_activity values (2)");
HRN_HOST_WAL_SWITCH(pg1);
HRN_HOST_SQL_EXEC(pg1, "insert into wal_activity values (3)");
HRN_HOST_WAL_SWITCH(pg1);
HRN_HOST_SQL_EXEC(pg1, "insert into wal_activity values (4)");
HRN_HOST_WAL_SWITCH(pg1);
TEST_STORAGE_EXISTS(
hrnHostDataStorage(pg1), zNewFmt("%s/" HRN_STANZA "-archive-push-async.log", strZ(hrnHostLogPath(pg1))));
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("setup time target");
// If the tests are running quickly then the time target might end up the same as the end time of the prior full backup.
// That means restore auto-select will not pick it as a candidate and restore the last backup instead causing the restore
// compare to fail. So, sleep one second.
sleepMSec(1000);
HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_TIME "'");
HRN_HOST_WAL_SWITCH(pg1);
const char *const targetTime = strZ(pckReadStrP(hrnHostSqlValue(pg1, "select current_timestamp::text")));
TEST_LOG_FMT("time target = %s", targetTime);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary incr delta backup");
{
// Create a database that can be excluded from restores
HRN_HOST_SQL_EXEC(pg1, "create database exclude_me with tablespace ts1");
// Check that backup fails for <= 9.5 when another backup is already running
if (hrnHostPgVersion() <= PG_VERSION_95)
{
HRN_HOST_SQL_EXEC(pg1, "perform pg_start_backup('test backup that will be restarted', true)");
TEST_HOST_BR(repo, CFGCMD_BACKUP, .resultExpect = errorTypeCode(&DbQueryError));
}
// Include stop auto here so backups for <= 9.5 will stop the prior backup
HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_INCR "'");
TEST_HOST_BR(repo, CFGCMD_BACKUP, .option = "--type=incr --delta --stop-auto");
}
// Get exclude_me database oid
const unsigned int excludeMeOid = pckReadU32P(
hrnHostSqlValue(pg1, "select oid from pg_database where datname = 'exclude_me'"));
TEST_LOG_FMT("exclude_me database oid = %u", excludeMeOid);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("setup xid target");
hrnHostSqlBegin(pg1);
HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_XID "'");
HRN_HOST_WAL_SWITCH(pg1);
const char *const targetXid = strZ(pckReadStrP(hrnHostSqlValue(pg1, "select txid_current()::text")));
TEST_LOG_FMT("xid target = %s", targetXid);
hrnHostSqlCommit(pg1);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("setup name target");
HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_NAME "'");
HRN_HOST_WAL_SWITCH(pg1);
HRN_HOST_SQL_EXEC(pg1, "perform pg_create_restore_point('" TEST_RESTORE_POINT "')");
TEST_LOG("name target = " TEST_RESTORE_POINT);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary restore (default target)");
{
// Stop the cluster
HRN_HOST_PG_STOP(pg1);
// Restore
TEST_HOST_BR(pg1, CFGCMD_RESTORE, .option = zNewFmt("--force --repo=%u", hrnHostRepoTotal()));
HRN_HOST_PG_START(pg1);
// Check that backup recovered to the expected target
TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_NAME);
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary restore (immediate target)");
{
// Expect failure when pg is running
TEST_HOST_BR(pg1, CFGCMD_RESTORE, .resultExpect = errorTypeCode(&PgRunningError));
// Stop the cluster and try again
HRN_HOST_PG_STOP(pg1);
// Restore
TEST_HOST_BR(pg1, CFGCMD_RESTORE, .option = "--delta --type=immediate --target-action=promote --db-exclude=exclude_me");
HRN_HOST_PG_START(pg1);
// Test that the exclude_me database has a zeroed pg_filenode.map
const Buffer *const pgFileNodeMap = storageGetP(
storageNewReadP(
hrnHostPgStorage(pg1),
strNewFmt(
PG_PATH_PGTBLSPC "/%u/%s/%u/" PG_FILE_PGFILENODEMAP, ts1Oid,
strZ(pgTablespaceId(hrnHostPgVersion(), hrnPgCatalogVersion(hrnHostPgVersion()))), excludeMeOid)));
Buffer *const pgFileNodeMapZero = bufNew(bufUsed(pgFileNodeMap));
memset(bufPtr(pgFileNodeMapZero), 0, bufSize(pgFileNodeMapZero));
bufUsedSet(pgFileNodeMapZero, bufSize(pgFileNodeMapZero));
TEST_RESULT_BOOL(bufEq(pgFileNodeMap, pgFileNodeMapZero), true, "exclude_me db " PG_FILE_PGFILENODEMAP " is zeroed");
// Check that backup recovered to the expected target
TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_INCR);
// It should be possible to drop the exclude_me database even though it was not restored
HRN_HOST_SQL_EXEC(pg1, "drop database exclude_me");
}
// From here on restores need to specify the current timeline for > 11 for recovery to be reliable
const char *const targetTimeline = hrnHostPgVersion() <= PG_VERSION_11 ? "" : " --target-timeline=current";
// Be careful about moving the order of this restore since the timeline created is used in a later test
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary restore (xid target)");
{
// Stop the cluster
HRN_HOST_PG_STOP(pg1);
// Restore
TEST_HOST_BR(
pg1, CFGCMD_RESTORE,
.option = zNewFmt(
"--delta --type=xid --target=%s --target-action=promote --repo=%u%s", targetXid, hrnHostRepoTotal(),
targetTimeline));
HRN_HOST_PG_START(pg1);
// Check that backup recovered to the expected target
TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_XID);
// Update status to test following a specified timeline
HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_TIMELINE "'");
HRN_HOST_WAL_SWITCH(pg1);
}
// Store the time so it can be used in a later test
const char *const xidTimeline = strZ(
pckReadStrP(
hrnHostSqlValue(
pg1,
zNewFmt(
"select trim(leading '0' from substring(pg_%sfile_name('1/1'), 1, 8))",
strZ(pgWalName(hrnHostPgVersion()))))));
TEST_LOG_FMT("xid timeline = %s", xidTimeline);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary restore (time target, auto-select backup)");
{
// Stop the cluster
HRN_HOST_PG_STOP(pg1);
// Restore
TEST_HOST_BR(
pg1, CFGCMD_RESTORE,
.option = zNewFmt("--delta --type=time --target='%s'%s", targetTime, targetTimeline));
HRN_HOST_PG_START(pg1);
// Check that backup recovered to the expected target
TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_TIME);
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary restore (time xid, exclusive)");
{
// Stop the cluster
HRN_HOST_PG_STOP(pg1);
// Restore
TEST_HOST_BR(
pg1, CFGCMD_RESTORE,
.option = zNewFmt(
"--delta --type=xid --target='%s' --target-exclusive --repo=%u%s", targetXid, hrnHostRepoTotal(),
targetTimeline));
HRN_HOST_PG_START(pg1);
// Check that backup recovered to the expected target
TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_INCR);
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary restore (name)");
{
// Stop the cluster
HRN_HOST_PG_STOP(pg1);
// Restore
TEST_HOST_BR(
pg1, CFGCMD_RESTORE, .option = zNewFmt("--delta --type=name --target='" TEST_RESTORE_POINT "'%s", targetTimeline));
HRN_HOST_PG_START(pg1);
// Check that backup recovered to the expected target
TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_NAME);
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("primary restore (default, timeline created by type = xid)");
{
// Stop the cluster
HRN_HOST_PG_STOP(pg1);
TEST_HOST_BR(pg1, CFGCMD_RESTORE, .option = zNewFmt("--delta --type=standby --target-timeline=%s", xidTimeline));
HRN_HOST_PG_START(pg1);
// Check that backup recovered to the expected target
TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_TIMELINE);
}
// -------------------------------------------------------------------------------------------------------------------------
if (hrnHostNonVersionSpecific())
{
TEST_TITLE("stanza-delete --force with pgbackrest stopped");
// Stop the cluster
HRN_HOST_PG_STOP(pg1);
// Stop pgbackrest
TEST_HOST_BR(pg1, CFGCMD_STOP);
TEST_HOST_BR(repo, CFGCMD_STOP);
// Delete stanza
TEST_HOST_BR(repo, CFGCMD_STANZA_DELETE, .option = "--force");
}
}
FUNCTION_HARNESS_RETURN_VOID();
}

174
test/src/module/test/testTest.c
View File

@ -152,6 +152,7 @@ testRun(void)
" feature: stackTrace\n"
" harness:\n"
" name: stackTrace\n"
" integration: false\n"
" shim:\n"
" common/stackTrace:\n"
" function:\n"
@ -196,7 +197,7 @@ testRun(void)
" db: true\n"
" test:\n"
" - name: all\n"
" total: 2\n"
" total: 1\n"
"\n"
"performance:\n"
" - name: performance\n"
@ -241,7 +242,7 @@ testRun(void)
TEST_RESULT_VOID(
cmdTest(
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3,
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"),
STRDEF("common/stack-trace"), 0, 1, logLevelDebug, true, NULL, false, false, false, true),
"new build");
@ -326,6 +327,8 @@ testRun(void)
String *const testCDup = strCat(strNew(), testC);
strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false"));
strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("none"));
strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid"));
strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true"));
strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE"));
strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build"));
@ -366,7 +369,7 @@ testRun(void)
TEST_RESULT_VOID(
cmdTest(
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3,
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"),
STRDEF("common/error"), 5, 1, logLevelDebug, true, NULL, false, false, false, true),
"new build");
@ -443,6 +446,8 @@ testRun(void)
String *const testCDup = strCat(strNew(), testC);
strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false"));
strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("none"));
strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid"));
strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true"));
strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE"));
strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build"));
@ -561,7 +566,7 @@ testRun(void)
TEST_RESULT_VOID(
cmdTest(
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3,
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"),
STRDEF("test/shim"), 0, 1, logLevelDebug, true, NULL, true, true, true, true),
"new build");
@ -671,6 +676,8 @@ testRun(void)
String *const testCDup = strCat(strNew(), testC);
strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true"));
strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX"));
strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid"));
strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true"));
strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled"));
strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build"));
@ -706,10 +713,157 @@ testRun(void)
TEST_RESULT_VOID(
cmdTest(
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3,
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"),
STRDEF("test/shim"), 0, 1, logLevelDebug, true, NULL, true, true, true, true),
"new build");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("Test real/all");
HRN_STORAGE_PUT_Z(
storageTest, "repo/test/src/module/real/allTest.c",
"static void\n"
"testRun(void)\n"
"{\n"
" FUNCTION_HARNESS_VOID();\n"
"\n"
" if (testBegin(\"all\"))\n"
" {\n"
" }\n"
"\n"
" FUNCTION_HARNESS_RETURN_VOID();\n"
"}\n");
TEST_RESULT_VOID(
cmdTest(
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"),
STRDEF("real/all"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), false, true, false, true),
"new build");
storageUnit = storagePosixNewP(STRDEF(TEST_PATH "/test/unit-3/none"));
fileList = testStorageList(storageUnit);
TEST_RESULT_STRLST_Z(
fileList,
"meson.build\n"
"meson_options.txt\n"
"test/src/common/harnessError.c\n"
"test/src/common/harnessShim.c\n"
"test/src/common/shim.c\n"
"test.c\n",
"check files");
for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++)
{
const String *const file = strLstGet(fileList, fileIdx);
if (strEqZ(file, "meson.build"))
{
TEST_STORAGE_GET(
storageUnit, strZ(file),
zNewFmt(
"%s"
"add_global_arguments('-DHRN_FEATURE_ERROR', language : 'c')\n"
"add_global_arguments('-DHRN_FEATURE_STACKTRACE', language : 'c')\n"
"\n"
MESON_COMMENT_BLOCK "\n"
"# Unit test\n"
MESON_COMMENT_BLOCK "\n"
"src_unit = files(\n"
" '../../../repo/src/common/stackTrace.c',\n"
" '../../../repo/src/common/type/stringStatic.c',\n"
" '../../../repo/src/common/debug.c',\n"
" 'test/src/common/harnessError.c',\n"
" '../../../repo/test/src/common/harnessNoShim.c',\n"
" 'test/src/common/harnessShim.c',\n"
" '../../../repo/test/src/common/harnessTest.c',\n"
" 'test.c',\n"
")\n"
"\n"
"executable(\n"
" 'test-unit',\n"
" sources: src_unit,\n"
" c_args: [\n"
" '-pg',\n"
" '-no-pie',\n"
" ],\n"
" link_args: [\n"
" '-pg',\n"
" '-no-pie',\n"
" ],\n"
" include_directories:\n"
" include_directories(\n"
" '.',\n"
" '../../../repo/src',\n"
" '../../../repo/doc/src',\n"
" '../../../repo/test/src',\n"
" ),\n"
" dependencies: [\n"
" lib_backtrace,\n"
" lib_bz2,\n"
" lib_openssl,\n"
" lib_lz4,\n"
" lib_pq,\n"
" lib_ssh2,\n"
" lib_xml,\n"
" lib_yaml,\n"
" lib_z,\n"
" lib_zstd,\n"
" ],\n"
")\n",
strZ(mesonBuildRoot)));
}
else if (strEqZ(file, "meson_options.txt"))
{
TEST_STORAGE_GET(storageUnit, strZ(file), mesonOption);
}
else if (strEqZ(file, "src/common/stackTrace.c") || strEqZ(file, "test/src/common/harnessStackTrace.c"))
{
// No test needed
}
else if (strEqZ(file, "test/src/common/harnessError.c"))
{
TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessErrorC));
}
else if (strEqZ(file, "test/src/common/harnessShim.c"))
{
String *const harnessShimCInt = strCat(strNew(), harnessShimC);
strReplace(harnessShimCInt, STRDEF("uXX"), STRDEF("none"));
TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessShimCInt));
}
else if (strEqZ(file, "test/src/common/shim.c"))
{
TEST_STORAGE_GET(storageUnit, strZ(file), strZ(shimC));
}
else if (strEqZ(file, "test.c"))
{
String *const testCDup = strCat(strNew(), testC);
strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false"));
strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX"));
strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid"));
strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false"));
strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled"));
strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build"));
strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("true"));
strReplace(testCDup, STRDEF("{[C_TEST_PROJECT_EXE]}"), STRDEF(TEST_PATH "/test/bin/uXX/pgbackrest"));
strReplace(testCDup, STRDEF("{[C_TEST_TZ]}"), STRDEF("setenv(\"TZ\", \"America/New_York\", true);"));
strReplace(testCDup, STRDEF("{[C_INCLUDE]}"), STRDEF(""));
strReplace(
testCDup, STRDEF("{[C_TEST_INCLUDE]}"), STRDEF("#include \"../../../repo/test/src/module/real/allTest.c\""));
strReplace(
testCDup, STRDEF("{[C_TEST_LIST]}"),
STRDEF(
"hrnAdd( 1, true);"));
TEST_STORAGE_GET(storageUnit, strZ(file), strZ(testCDup));
}
else
THROW_FMT(TestError, "no test for '%s'", strZ(file));
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("Test performance/type");
@ -735,7 +889,7 @@ testRun(void)
TEST_RESULT_VOID(
cmdTest(
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3,
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"),
STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), false, true, false, false),
"new build");
@ -845,6 +999,8 @@ testRun(void)
String *const testCDup = strCat(strNew(), testC);
strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true"));
strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX"));
strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid"));
strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false"));
strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled"));
strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build"));
@ -872,7 +1028,7 @@ testRun(void)
TEST_RESULT_VOID(
cmdTest(
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3,
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"),
STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), false, false, false, false),
"new build");
@ -885,6 +1041,8 @@ testRun(void)
String *const testCDup = strCat(strNew(), testC);
strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true"));
strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX"));
strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid"));
strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false"));
strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled"));
strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build"));
@ -910,7 +1068,7 @@ testRun(void)
TEST_ERROR(
cmdTest(
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3,
STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"),
STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), false, false, false, false),
FileOpenError,
"build failed for unit performance/type: unable to open file '" TEST_PATH "/repo/meson.build' for read: [13] Permission"

9
test/src/test.c
View File

@ -79,6 +79,12 @@ STRING_EXTERN(HRN_PATH_STR, HRN_PATH);
#define TEST_USER_ID_Z "{[C_TEST_USER_ID]}"
#define TEST_USER_LEN "{[C_TEST_USER_LEN]}"
// VM for integration testing
#define TEST_VM "{[C_TEST_VM]}"
// PostgreSQL version for integration testing
#define TEST_PG_VERSION "{[C_TEST_PG_VERSION]}"
#ifdef HRN_FEATURE_STRING
STRING_EXTERN(TEST_USER_STR, TEST_USER);
#endif
@ -203,6 +209,9 @@ main(int argListSize, const char *argList[])
{[C_TEST_IDX]}, // The 0-based index of this test
{[C_TEST_TIMING]}, // Is timing enabled (may be disabled for reproducible documentation)
TEST_PATH, // Path where tests write data
TEST_USER, // User running the test
TEST_VM, // VM for integration testing
TEST_PG_VERSION, // PostgreSQL version for integration testing
HRN_PATH, // Path where the harness stores temp files (expect, diff, etc.)
HRN_PATH_REPO); // Path with a copy of the repository

16
test/src/valgrind.suppress.none Normal file
View File

@ -0,0 +1,16 @@
# Suppress issues in libssh2 on Ubuntu 20.04
{
libssh2_ubuntu_20_04
Memcheck:Addr1
...
obj:/usr/lib/x86_64-linux-gnu/libssh2.so.1.0.1
...
}
{
libssh2_ubuntu_20_04
Memcheck:Leak
match-leak-kinds: definite
...
obj:/usr/lib/x86_64-linux-gnu/libssh2.so.1.0.1
...
}

1375
test/test.pl
View File

File diff suppressed because it is too large Load Diff