db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-05 00:28:52 +02:00
Code Issues Releases Activity

Make storage type more generic in the integration tests.

Browse Source 
Rather than bS3 use strStorage which can indicate more than two storage types.

For the moment there are still only two storage types but this change is required before more can be added.
This commit is contained in:
David Steele
2020-05-12 18:55:20 -04:00
parent 9639a2c15f
commit b5dd14e6f3
24 changed files with 160 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
View File

@ -205,14 +205,14 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => LZ4},
{vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => BZ2},
{vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => BZ2},
{vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => ZST},
{vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => LZ4},
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => GZ},
{vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => ZST},
{vm => VM1, remote => false, storage => S3, encrypt => false, delta => true, compress => LZ4},
{vm => VM1, remote => true, storage => POSIX, encrypt => true, delta => false, compress => BZ2},
{vm => VM2, remote => false, storage => POSIX, encrypt => true, delta => true, compress => BZ2},
{vm => VM2, remote => true, storage => S3, encrypt => false, delta => false, compress => GZ},
{vm => VM3, remote => false, storage => POSIX, encrypt => false, delta => true, compress => ZST},
{vm => VM3, remote => true, storage => S3, encrypt => true, delta => false, compress => LZ4},
{vm => VM4, remote => false, storage => POSIX, encrypt => false, delta => false, compress => GZ},
{vm => VM4, remote => true, storage => S3, encrypt => true, delta => true, compress => ZST},
)
{
# Only run tests for this vm
@ -220,21 +220,22 @@ sub run
# Increment the run, log, and decide whether this unit test should be run
my $bRemote = $rhRun->{remote};
my $bS3 = $rhRun->{s3};
my $strStorage = $rhRun->{storage};
my $bEncrypt = $rhRun->{encrypt};
my $bDeltaBackup = $rhRun->{delta};
my $strCompressType = $rhRun->{compress};
# Increment the run, log, and decide whether this unit test should be run
if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, delta ${bDeltaBackup}")) {next}
if (!$self->begin("rmt ${bRemote}, storage ${strStorage}, enc ${bEncrypt}, delta ${bDeltaBackup}")) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE});
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, strStorage => $strStorage, bRepoEncrypt => $bEncrypt,
strCompressType => NONE});
# If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing
# without slowing down the other tests too much.
if ($bS3)
if ($strStorage eq S3)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
@ -263,7 +264,7 @@ sub run
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK} = $bRemote ? 1 : 3;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::false;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} = JSON::PP::false;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = JSON::PP::false;
if ($bEncrypt)
@ -492,14 +493,14 @@ sub run
# Pass bogus socket path to make sure it is passed through the protocol layer (it won't be used)
($bRemote ? ' --pg1-socket-path=/test_socket_path' : '') .
' --buffer-size=16384 --checksum-page --process-max=1',
strRepoType => $bS3 ? undef : STORAGE_CIFS, strTest => $strTestPoint, fTestDelay => 0});
strRepoType => $strStorage eq S3 ? undef : CIFS, strTest => $strTestPoint, fTestDelay => 0});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 65536;
# Stop operations and make sure the correct error occurs
#---------------------------------------------------------------------------------------------------------------------------
if (!$bS3)
if ($strStorage eq POSIX)
{
# Test global stop
$oHostDbMaster->stop({bForce => true});
@ -582,7 +583,7 @@ sub run
# Create a temp file in backup temp root to be sure it's deleted correctly
my $strTempFile = "${strResumePath}/file.tmp";
if ($bS3)
if ($strStorage eq S3)
{
storageRepo()->put($strTempFile, "TEMP");
}
@ -842,7 +843,7 @@ sub run
{oExpectedManifest => \%oManifest,
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1;
if (!$bRemote)
{
@ -863,7 +864,7 @@ sub run
$strType, 'drop tablespace 11',
{oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1;
# Restore
#---------------------------------------------------------------------------------------------------------------------------
@ -918,7 +919,7 @@ sub run
$strType, 'add files and remove tablespace 2',
{oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1'});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1;
# Incr Backup
#---------------------------------------------------------------------------------------------------------------------------
@ -943,7 +944,7 @@ sub run
$strType, 'updates since last full', {oExpectedManifest => \%oManifest,
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1;
# Diff Backup with files removed
#---------------------------------------------------------------------------------------------------------------------------
@ -962,7 +963,7 @@ sub run
}
# Enable hardlinks (except for s3) to ensure a warning is raised
if (!$bS3)
if ($strStorage eq POSIX)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'repo1-hardlink' => 'y'}});
}
@ -982,7 +983,7 @@ sub run
{oExpectedManifest => \%oManifest,
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1;
# Full Backup
#---------------------------------------------------------------------------------------------------------------------------
@ -992,7 +993,7 @@ sub run
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = $strCompressType;
if (!$bS3)
if ($strStorage eq POSIX)
{
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::true;
$oHostBackup->{bHardLink} = true;
@ -1102,7 +1103,7 @@ sub run
# because for some reason sort order is different when this command is executed via ssh (even though the content of the
# directory is identical).
#---------------------------------------------------------------------------------------------------------------------------
if (!$bRemote && !$bS3)
if (!$bRemote && $strStorage eq POSIX)
{
executeTest(
'ls -1Rtr ' . $oHostBackup->repoBackupPath(PATH_BACKUP_HISTORY),

24
test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm
View File

@ -43,14 +43,14 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4, error => 0},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ, error => 1},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => NONE, error => 0},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => BZ2, error => 0},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => LZ4, error => 1},
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST, error => 0},
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => NONE, error => 0},
{vm => VM1, remote => false, storage => POSIX, encrypt => false, compress => LZ4, error => 0},
{vm => VM1, remote => true, storage => S3, encrypt => true, compress => GZ, error => 1},
{vm => VM2, remote => false, storage => S3, encrypt => false, compress => NONE, error => 0},
{vm => VM2, remote => true, storage => POSIX, encrypt => true, compress => BZ2, error => 0},
{vm => VM3, remote => false, storage => POSIX, encrypt => true, compress => NONE, error => 0},
{vm => VM3, remote => true, storage => S3, encrypt => false, compress => LZ4, error => 1},
{vm => VM4, remote => false, storage => S3, encrypt => true, compress => ZST, error => 0},
{vm => VM4, remote => true, storage => POSIX, encrypt => false, compress => NONE, error => 0},
)
{
# Only run tests for this vm
@ -58,7 +58,7 @@ sub run
# Increment the run, log, and decide whether this unit test should be run
my $bRemote = $rhRun->{remote};
my $bS3 = $rhRun->{s3};
my $strStorage = $rhRun->{storage};
my $bEncrypt = $rhRun->{encrypt};
my $strCompressType = $rhRun->{compress};
my $iError = $rhRun->{error};
@ -66,12 +66,12 @@ sub run
# Increment the run, log, and decide whether this unit test should be run
if (!$self->begin(
"rmt ${bRemote}, cmp ${strCompressType}, error " . ($iError ? 'connect' : 'version') .
", s3 ${bS3}, enc ${bEncrypt}")) {next}
", storage ${strStorage}, enc ${bEncrypt}")) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, strCompressType => $strCompressType, bArchiveAsync => true,
bS3 => $bS3, bRepoEncrypt => $bEncrypt});
strStorage => $strStorage, bRepoEncrypt => $bEncrypt});
# Create compression extension
my $strCompressExt = $strCompressType ne NONE ? ".${strCompressType}" : '';

29
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
View File

@ -83,14 +83,14 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => LZ4},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => BZ2},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => BZ2},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => LZ4},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => ZST},
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST},
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => GZ},
{vm => VM1, remote => false, storage => POSIX, encrypt => false, compress => GZ},
{vm => VM1, remote => true, storage => S3, encrypt => true, compress => LZ4},
{vm => VM2, remote => false, storage => S3, encrypt => false, compress => BZ2},
{vm => VM2, remote => true, storage => POSIX, encrypt => true, compress => BZ2},
{vm => VM3, remote => false, storage => POSIX, encrypt => true, compress => LZ4},
{vm => VM3, remote => true, storage => S3, encrypt => false, compress => ZST},
{vm => VM4, remote => false, storage => S3, encrypt => true, compress => ZST},
{vm => VM4, remote => true, storage => POSIX, encrypt => false, compress => GZ},
)
{
# Only run tests for this vm
@ -98,15 +98,16 @@ sub run
# Increment the run, log, and decide whether this unit test should be run
my $bRemote = $rhRun->{remote};
my $bS3 = $rhRun->{s3};
my $strStorage = $rhRun->{storage};
my $bEncrypt = $rhRun->{encrypt};
my $strCompressType = $rhRun->{compress};
if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, cmp ${strCompressType}")) {next}
if (!$self->begin("rmt ${bRemote}, storage ${strStorage}, enc ${bEncrypt}, cmp ${strCompressType}")) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE});
true, $self->expect(), {bHostBackup => $bRemote, strStorage => $strStorage, bRepoEncrypt => $bEncrypt,
strCompressType => NONE});
# Reduce console logging to detail
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(DETAIL)}});
@ -205,7 +206,7 @@ sub run
# Create a temp file to make sure it is deleted later (skip when S3 since it doesn't use temp files)
my $strArchiveTmp;
if (!$bS3)
if ($strStorage eq POSIX)
{
# Should succeed when temp file already exists
&log(INFO, ' succeed when tmp WAL file exists');
@ -341,8 +342,8 @@ sub run
&log(INFO, " get second WAL (${strSourceFile})");
$oHostDbMaster->executeSimple(
$strCommandGet . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . " --archive-async" . (!$bS3 ? " --repo-type=cifs" : '') .
" --archive-timeout=5 ${strSourceFile} ${strWalPath}/RECOVERYXLOG",
$strCommandGet . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . " --archive-async" .
($strStorage eq POSIX ? " --repo-type=cifs" : '') . " --archive-timeout=5 ${strSourceFile} ${strWalPath}/RECOVERYXLOG",
{oLogTest => $self->expect()});
# Check that the destination file exists

26
test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm
View File

@ -45,30 +45,30 @@ sub run
foreach my $rhRun
(
{vm => VM1, s3 => true, encrypt => false},
{vm => VM2, s3 => false, encrypt => true},
{vm => VM3, s3 => false, encrypt => false},
{vm => VM4, s3 => true, encrypt => true},
{vm => VM1, storage => POSIX, encrypt => false},
{vm => VM2, storage => S3, encrypt => true},
{vm => VM3, storage => POSIX, encrypt => false},
{vm => VM4, storage => S3, encrypt => true},
)
{
# Only run tests for this vm
next if ($rhRun->{vm} ne vmTest($self->vm()));
# Increment the run, log, and decide whether this unit test should be run
my $bS3 = $rhRun->{s3};
my $strStorage = $rhRun->{storage};
my $bEncrypt = $rhRun->{encrypt};
if ($bS3 && ($self->vm() eq VM3))
if ($strStorage ne POSIX && ($self->vm() eq VM3))
{
confess &log("cannot configure s3 for expect log tests");
confess &log("cannot configure non-posix storage for expect log tests");
}
############################################################################################################################
if ($self->begin("simple, enc ${bEncrypt}, s3 ${bS3}"))
if ($self->begin("simple, enc ${bEncrypt}, storage ${strStorage}"))
{
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
true, $self->expect(), {bS3 => $bS3, bRepoEncrypt => $bEncrypt});
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {strStorage => $strStorage, bRepoEncrypt => $bEncrypt});
# Create the test object
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest(
@ -156,11 +156,11 @@ sub run
}
############################################################################################################################
if ($self->begin("stanzaUpgrade, enc ${bEncrypt}, s3 ${bS3}"))
if ($self->begin("stanzaUpgrade, enc ${bEncrypt}, storage ${strStorage}"))
{
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
true, $self->expect(), {bS3 => $bS3, bRepoEncrypt => $bEncrypt});
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {strStorage => $strStorage, bRepoEncrypt => $bEncrypt});
# Create the test object
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest(

24
test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm
View File

@ -41,14 +41,14 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => true, compress => LZ4},
{vm => VM1, remote => true, s3 => true, encrypt => false, compress => BZ2},
{vm => VM2, remote => false, s3 => true, encrypt => true, compress => BZ2},
{vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => false, compress => ZST},
{vm => VM3, remote => true, s3 => true, encrypt => true, compress => LZ4},
{vm => VM4, remote => false, s3 => true, encrypt => false, compress => GZ},
{vm => VM4, remote => true, s3 => false, encrypt => true, compress => ZST},
{vm => VM1, remote => false, storage => POSIX, encrypt => true, compress => LZ4},
{vm => VM1, remote => true, storage => S3, encrypt => false, compress => BZ2},
{vm => VM2, remote => false, storage => S3, encrypt => true, compress => BZ2},
{vm => VM2, remote => true, storage => POSIX, encrypt => false, compress => GZ},
{vm => VM3, remote => false, storage => POSIX, encrypt => false, compress => ZST},
{vm => VM3, remote => true, storage => S3, encrypt => true, compress => LZ4},
{vm => VM4, remote => false, storage => S3, encrypt => false, compress => GZ},
{vm => VM4, remote => true, storage => POSIX, encrypt => true, compress => ZST},
)
{
# Only run tests for this vm
@ -56,16 +56,16 @@ sub run
# Increment the run, log, and decide whether this unit test should be run
my $bRemote = $rhRun->{remote};
my $bS3 = $rhRun->{s3};
my $strStorage = $rhRun->{storage};
my $bEncrypt = $rhRun->{encrypt};
my $strCompressType = $rhRun->{compress};
# Increment the run, log, and decide whether this unit test should be run
if (!$self->begin("remote ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, cmp ${strCompressType}")) {next}
if (!$self->begin("remote ${bRemote}, storage ${strStorage}, enc ${bEncrypt}, cmp ${strCompressType}")) {next}
# Create hosts, file object, and config
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt,
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
true, $self->expect(), {bHostBackup => $bRemote, strStorage => $strStorage, bRepoEncrypt => $bEncrypt,
strCompressType => $strCompressType});
# Archive and backup info file names