You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2026-05-22 10:15:16 +02:00
Add infrastructure for multiple compression type support.
Add compress-type option and deprecate compress option. Since the compress option is boolean it won't work with multiple compression types. Add logic to cfgLoadUpdateOption() to update compress-type if it is not set directly. The compress option should no longer be referenced outside the cfgLoadUpdateOption() function. Add common/compress/helper module to contain interface functions that work with multiple compression types. Code outside this module should no longer call specific compression drivers, though it may be OK to reference a specific compression type using the new interface (e.g., saving backup history files in gz format). Unit tests only test compression using the gz format because other formats may not be available in all builds. It is the job of integration tests to exercise all compression types. Additional compression types will be added in future commits.
This commit is contained in:
David Steele
@@ -476,9 +476,9 @@ sub backupCompare
|
||||
foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE))
|
||||
{
|
||||
# Determine repo size if compression or encryption is enabled
|
||||
my $bCompressed = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS};
|
||||
my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE};
|
||||
|
||||
if ($bCompressed ||
|
||||
if ($strCompressType ne CFGOPTVAL_COMPRESS_TYPE_NONE ||
|
||||
(defined($oExpectedManifest->{&INI_SECTION_CIPHER}) &&
|
||||
defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS})))
|
||||
{
|
||||
@@ -487,7 +487,8 @@ sub backupCompare
|
||||
$oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
|
||||
$oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
|
||||
(storageRepo()->info(STORAGE_REPO_BACKUP .
|
||||
"/${strBackup}/${strFileKey}" . ($bCompressed ? '.gz' : '')))->{size};
|
||||
"/${strBackup}/${strFileKey}" .
|
||||
($strCompressType eq CFGOPTVAL_COMPRESS_TYPE_NONE ? '' : ".${strCompressType}")))->{size};
|
||||
|
||||
if (defined($lRepoSize) &&
|
||||
$lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
|
||||
@@ -1031,9 +1032,9 @@ sub configCreate
|
||||
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_LEVEL_NETWORK)} = 1;
|
||||
}
|
||||
|
||||
if (defined($$oParam{bCompress}) && !$$oParam{bCompress})
|
||||
if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne CFGOPTVAL_COMPRESS_TYPE_GZ)
|
||||
{
|
||||
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS)} = 'n';
|
||||
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_TYPE)} = $oParam->{strCompressType};
|
||||
}
|
||||
|
||||
if ($self->isHostBackup())
|
||||
@@ -1200,7 +1201,14 @@ sub configUpdate
|
||||
{
|
||||
foreach my $strKey (keys(%{$hParam->{$strSection}}))
|
||||
{
|
||||
$oConfig->{$strSection}{$strKey} = $hParam->{$strSection}{$strKey};
|
||||
if (defined($hParam->{$strSection}{$strKey}))
|
||||
{
|
||||
$oConfig->{$strSection}{$strKey} = $hParam->{$strSection}{$strKey};
|
||||
}
|
||||
else
|
||||
{
|
||||
delete($oConfig->{$strSection}{$strKey});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1792,6 +1800,8 @@ sub restoreCompare
|
||||
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}, $oTablespaceMap);
|
||||
$oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef,
|
||||
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA});
|
||||
$oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_TYPE, undef,
|
||||
$oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE});
|
||||
|
||||
my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH);
|
||||
|
||||
|
||||
@@ -43,6 +43,11 @@ use constant ENCRYPTION_KEY_MANIFEST => 'manifest';
|
||||
use constant ENCRYPTION_KEY_BACKUPSET => 'backupset';
|
||||
push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET);
|
||||
|
||||
use constant NONE => CFGOPTVAL_COMPRESS_TYPE_NONE;
|
||||
push @EXPORT, qw(NONE);
|
||||
use constant GZ => CFGOPTVAL_COMPRESS_TYPE_GZ;
|
||||
push @EXPORT, qw(GZ);
|
||||
|
||||
####################################################################################################################################
|
||||
# setup
|
||||
####################################################################################################################################
|
||||
@@ -127,7 +132,7 @@ sub setup
|
||||
# Create db master config
|
||||
$oHostDbMaster->configCreate({
|
||||
strBackupSource => $$oConfigParam{strBackupSource},
|
||||
bCompress => $$oConfigParam{bCompress},
|
||||
strCompressType => $$oConfigParam{strCompressType},
|
||||
bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink},
|
||||
bArchiveAsync => $$oConfigParam{bArchiveAsync},
|
||||
bS3 => $$oConfigParam{bS3}});
|
||||
@@ -136,7 +141,7 @@ sub setup
|
||||
if (defined($oHostBackup))
|
||||
{
|
||||
$oHostBackup->configCreate({
|
||||
bCompress => $$oConfigParam{bCompress},
|
||||
strCompressType => $$oConfigParam{strCompressType},
|
||||
bHardlink => $$oConfigParam{bHardLink},
|
||||
bS3 => $$oConfigParam{bS3}});
|
||||
}
|
||||
@@ -151,7 +156,7 @@ sub setup
|
||||
{
|
||||
$oHostDbStandby->configCreate({
|
||||
strBackupSource => $$oConfigParam{strBackupSource},
|
||||
bCompress => $$oConfigParam{bCompress},
|
||||
strCompressType => $$oConfigParam{strCompressType},
|
||||
bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink},
|
||||
bArchiveAsync => $$oConfigParam{bArchiveAsync}});
|
||||
}
|
||||
|
||||
@@ -62,14 +62,14 @@ sub run
|
||||
|
||||
foreach my $rhRun
|
||||
(
|
||||
{vm => VM1, remote => false, s3 => true, encrypt => false, delta => true},
|
||||
{vm => VM1, remote => true, s3 => false, encrypt => true, delta => false},
|
||||
{vm => VM2, remote => false, s3 => false, encrypt => true, delta => true},
|
||||
{vm => VM2, remote => true, s3 => true, encrypt => false, delta => false},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => false, delta => true},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => true, delta => false},
|
||||
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false},
|
||||
{vm => VM4, remote => true, s3 => true, encrypt => true, delta => true},
|
||||
{vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => GZ},
|
||||
{vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => GZ},
|
||||
{vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => GZ},
|
||||
{vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => GZ},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => GZ},
|
||||
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => GZ},
|
||||
{vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => GZ},
|
||||
)
|
||||
{
|
||||
# Only run tests for this vm
|
||||
@@ -80,13 +80,14 @@ sub run
|
||||
my $bS3 = $rhRun->{s3};
|
||||
my $bEncrypt = $rhRun->{encrypt};
|
||||
my $bDeltaBackup = $rhRun->{delta};
|
||||
my $strCompressType = $rhRun->{compress};
|
||||
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, delta ${bDeltaBackup}")) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => false, bS3 => $bS3, bRepoEncrypt => $bEncrypt});
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE});
|
||||
|
||||
# If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing
|
||||
# without slowing down the other tests too much.
|
||||
@@ -114,6 +115,7 @@ sub run
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 16384;
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE} = JSON::PP::true;
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::false;
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = CFGOPTVAL_COMPRESS_TYPE_NONE;
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL} = 3;
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK} = $bRemote ? 1 : 3;
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::false;
|
||||
@@ -809,8 +811,15 @@ sub run
|
||||
|
||||
$strType = CFGOPTVAL_BACKUP_TYPE_DIFF;
|
||||
|
||||
# Enable compression to ensure a warning is raised
|
||||
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS) => 'y'}});
|
||||
# Enable compression to ensure a warning is raised (reset when gz to avoid log churn since it is the default)
|
||||
if ($strCompressType eq GZ)
|
||||
{
|
||||
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => undef}});
|
||||
}
|
||||
else
|
||||
{
|
||||
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => $strCompressType}});
|
||||
}
|
||||
|
||||
# Enable hardlinks (except for s3) to ensure a warning is raised
|
||||
if (!$bS3)
|
||||
@@ -841,6 +850,7 @@ sub run
|
||||
|
||||
# Now the compression and hardlink changes will take effect
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::true;
|
||||
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = $strCompressType;
|
||||
|
||||
if (!$bS3)
|
||||
{
|
||||
|
||||
@@ -44,14 +44,14 @@ sub run
|
||||
|
||||
foreach my $rhRun
|
||||
(
|
||||
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => true, error => 0},
|
||||
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => false, error => 1},
|
||||
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => false, error => 0},
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => true, error => 0},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => false, error => 0},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => true, error => 1},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => true, error => 0},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => false, error => 0},
|
||||
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ, error => 0},
|
||||
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ, error => 1},
|
||||
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => NONE, error => 0},
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ, error => 0},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => GZ, error => 1},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => GZ, error => 0},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => NONE, error => 0},
|
||||
)
|
||||
{
|
||||
# Only run tests for this vm
|
||||
@@ -61,20 +61,21 @@ sub run
|
||||
my $bRemote = $rhRun->{remote};
|
||||
my $bS3 = $rhRun->{s3};
|
||||
my $bEncrypt = $rhRun->{encrypt};
|
||||
my $bCompress = $rhRun->{compress};
|
||||
my $strCompressType = $rhRun->{compress};
|
||||
my $iError = $rhRun->{error};
|
||||
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version') . ", s3 ${bS3}, " .
|
||||
"enc ${bEncrypt}")) {next}
|
||||
if (!$self->begin(
|
||||
"rmt ${bRemote}, cmp ${strCompressType}, error " . ($iError ? 'connect' : 'version') .
|
||||
", s3 ${bS3}, enc ${bEncrypt}")) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true, bS3 => $bS3,
|
||||
bRepoEncrypt => $bEncrypt});
|
||||
true, $self->expect(), {bHostBackup => $bRemote, strCompressType => $strCompressType, bArchiveAsync => true,
|
||||
bS3 => $bS3, bRepoEncrypt => $bEncrypt});
|
||||
|
||||
# Create compression extension
|
||||
my $strCompressExt = $bCompress ? qw{.} . COMPRESS_EXT : '';
|
||||
my $strCompressExt = $strCompressType ne NONE ? ".${strCompressType}" : '';
|
||||
|
||||
# Create the wal path
|
||||
my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
|
||||
|
||||
@@ -40,15 +40,15 @@ sub archiveCheck
|
||||
my $self = shift;
|
||||
my $strArchiveFile = shift;
|
||||
my $strArchiveChecksum = shift;
|
||||
my $bCompress = shift;
|
||||
my $strCompressType = shift;
|
||||
my $strSpoolPath = shift;
|
||||
|
||||
# Build the archive name to check for at the destination
|
||||
my $strArchiveCheck = PG_VERSION_94 . "-1/${strArchiveFile}-${strArchiveChecksum}";
|
||||
|
||||
if ($bCompress)
|
||||
if (defined($strCompressType))
|
||||
{
|
||||
$strArchiveCheck .= '.gz';
|
||||
$strArchiveCheck .= ".${strCompressType}";
|
||||
}
|
||||
|
||||
my $oWait = waitInit(5);
|
||||
@@ -82,14 +82,14 @@ sub run
|
||||
|
||||
foreach my $rhRun
|
||||
(
|
||||
{vm => VM1, remote => false, s3 => false, encrypt => false},
|
||||
{vm => VM1, remote => true, s3 => true, encrypt => true},
|
||||
{vm => VM2, remote => false, s3 => true, encrypt => false},
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => true},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => true},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => false},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => true},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => false},
|
||||
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ},
|
||||
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ},
|
||||
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => GZ},
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => GZ},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => GZ},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => GZ},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => GZ},
|
||||
)
|
||||
{
|
||||
# Only run tests for this vm
|
||||
@@ -99,12 +99,13 @@ sub run
|
||||
my $bRemote = $rhRun->{remote};
|
||||
my $bS3 = $rhRun->{s3};
|
||||
my $bEncrypt = $rhRun->{encrypt};
|
||||
my $strCompressType = $rhRun->{compress};
|
||||
|
||||
if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bEncrypt}")) {next}
|
||||
if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, cmp ${strCompressType}")) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bCompress => false, bS3 => $bS3, bRepoEncrypt => $bEncrypt});
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE});
|
||||
|
||||
# Reduce console logging to detail
|
||||
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(DETAIL)}});
|
||||
@@ -153,12 +154,13 @@ sub run
|
||||
my $strArchiveFile = $self->walGenerate($strWalPath, PG_VERSION_94, 2, $strSourceFile);
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandPush . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . " --compress ${strWalPath}/${strSourceFile}",
|
||||
$strCommandPush . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') .
|
||||
" --compress-type=${strCompressType} ${strWalPath}/${strSourceFile}",
|
||||
{oLogTest => $self->expect()});
|
||||
push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.gz");
|
||||
push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}");
|
||||
|
||||
# Test that the WAL was pushed
|
||||
$self->archiveCheck($strSourceFile, $strArchiveChecksum, true);
|
||||
$self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType);
|
||||
|
||||
# Remove from archive_status
|
||||
storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready");
|
||||
@@ -211,7 +213,7 @@ sub run
|
||||
|
||||
$strArchiveTmp =
|
||||
$oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' .
|
||||
substr($strSourceFile, 0, 16) . "/${strSourceFile}-${strArchiveChecksum}." . COMPRESS_EXT . qw{.} .
|
||||
substr($strSourceFile, 0, 16) . "/${strSourceFile}-${strArchiveChecksum}.${strCompressType}" . qw{.} .
|
||||
STORAGE_TEMP_EXT;
|
||||
|
||||
storageTest()->put($strArchiveTmp, 'JUNK');
|
||||
@@ -219,9 +221,10 @@ sub run
|
||||
|
||||
# Push the WAL
|
||||
$oHostDbMaster->executeSimple(
|
||||
"${strCommandPush} --compress --archive-async --process-max=2 ${strWalPath}/${strSourceFile}",
|
||||
"${strCommandPush} --compress-type=${strCompressType} --archive-async --process-max=2" .
|
||||
" ${strWalPath}/${strSourceFile}",
|
||||
{oLogTest => $self->expect()});
|
||||
push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}." . COMPRESS_EXT);
|
||||
push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}");
|
||||
|
||||
# Make sure the temp file no longer exists if it was created
|
||||
if (defined($strArchiveTmp))
|
||||
@@ -242,7 +245,7 @@ sub run
|
||||
}
|
||||
|
||||
# Test that the WAL was pushed
|
||||
$self->archiveCheck($strSourceFile, $strArchiveChecksum, true, $oHostDbMaster->spoolPath());
|
||||
$self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbMaster->spoolPath());
|
||||
|
||||
# Remove from archive_status
|
||||
storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready");
|
||||
@@ -381,7 +384,7 @@ sub run
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandPush . " ${strWalPath}/${strSourceFile}.partial",
|
||||
{oLogTest => $self->expect()});
|
||||
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum, false);
|
||||
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum);
|
||||
|
||||
push(@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}");
|
||||
|
||||
@@ -390,8 +393,7 @@ sub run
|
||||
|
||||
$oHostDbMaster->executeSimple(
|
||||
$strCommandPush . " ${strWalPath}/${strSourceFile}.partial", {oLogTest => $self->expect()});
|
||||
$self->archiveCheck(
|
||||
"${strSourceFile}.partial", $strArchiveChecksum, false);
|
||||
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum);
|
||||
|
||||
#---------------------------------------------------------------------------------------------------------------------------
|
||||
&log(INFO, ' .partial WAL with different checksum');
|
||||
|
||||
@@ -53,14 +53,14 @@ sub run
|
||||
|
||||
foreach my $rhRun
|
||||
(
|
||||
{vm => VM1, remote => false, s3 => false, encrypt => true},
|
||||
{vm => VM1, remote => true, s3 => true, encrypt => false},
|
||||
{vm => VM2, remote => false, s3 => true, encrypt => true},
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => false},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => false},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => true},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => false},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => true},
|
||||
{vm => VM1, remote => false, s3 => false, encrypt => true, compress => GZ},
|
||||
{vm => VM1, remote => true, s3 => true, encrypt => false, compress => GZ},
|
||||
{vm => VM2, remote => false, s3 => true, encrypt => true, compress => GZ},
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => false, compress => GZ},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => true, compress => GZ},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => false, compress => GZ},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => true, compress => GZ},
|
||||
)
|
||||
{
|
||||
# Only run tests for this vm
|
||||
@@ -70,13 +70,15 @@ sub run
|
||||
my $bRemote = $rhRun->{remote};
|
||||
my $bS3 = $rhRun->{s3};
|
||||
my $bEncrypt = $rhRun->{encrypt};
|
||||
my $strCompressType = $rhRun->{compress};
|
||||
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!$self->begin("remote ${bRemote}, s3 ${bS3}, enc ${bEncrypt}")) {next}
|
||||
if (!$self->begin("remote ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, cmp ${strCompressType}")) {next}
|
||||
|
||||
# Create hosts, file object, and config
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt});
|
||||
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt,
|
||||
strCompressType => $strCompressType});
|
||||
|
||||
# Create the stanza
|
||||
$oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_MISSING,
|
||||
@@ -192,7 +194,7 @@ sub run
|
||||
$oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1);
|
||||
$self->testResult(
|
||||
sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-2/0000000100000001')},
|
||||
'000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_94) . '.' . COMPRESS_EXT,
|
||||
'000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_94) . ".${strCompressType}",
|
||||
'check that WAL is in the archive at -2');
|
||||
|
||||
# Create the tablespace directory and perform a backup
|
||||
|
||||
@@ -62,15 +62,15 @@ sub run
|
||||
foreach my $strBackupDestination (
|
||||
$bS3 || $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER))
|
||||
{
|
||||
my $bCompress = $bHostBackup && !$bHostStandby;
|
||||
my $bRepoEncrypt = ($bCompress && !$bS3) ? true : false;
|
||||
my $strCompressType = $bHostBackup && !$bHostStandby ? GZ : NONE;
|
||||
my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false;
|
||||
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
my $hyVm = vmGet();
|
||||
my $strDbVersionMostRecent = ${$hyVm->{$self->vm()}{&VM_DB_TEST}}[-1];
|
||||
|
||||
next if (!$self->begin(
|
||||
"bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, cmp ${bCompress}, s3 ${bS3}, " .
|
||||
"bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, cmp ${strCompressType}, s3 ${bS3}, " .
|
||||
"enc ${bRepoEncrypt}",
|
||||
# Use the most recent db version on the expect vm for expect testing
|
||||
$self->vm() eq VM_EXPECT && $self->pgVersion() eq $strDbVersionMostRecent));
|
||||
@@ -100,7 +100,7 @@ sub run
|
||||
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
|
||||
false, $self->expect(),
|
||||
{bHostBackup => $bHostBackup, bStandby => $bHostStandby, strBackupDestination => $strBackupDestination,
|
||||
bCompress => $bCompress, bArchiveAsync => false, bS3 => $bS3, bRepoEncrypt => $bRepoEncrypt});
|
||||
strCompressType => $strCompressType, bArchiveAsync => false, bS3 => $bS3, bRepoEncrypt => $bRepoEncrypt});
|
||||
|
||||
# Only perform extra tests on certain runs to save time
|
||||
my $bTestLocal = $self->runCurrent() == 1;
|
||||
@@ -416,7 +416,7 @@ sub run
|
||||
|
||||
# Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it
|
||||
# down enough to make it evident that the async process is working.
|
||||
if ($bTestExtra && $bCompress && $strBackupDestination eq HOST_BACKUP)
|
||||
if ($bTestExtra && $strCompressType ne NONE && $strBackupDestination eq HOST_BACKUP)
|
||||
{
|
||||
&log(INFO, ' multiple wal switches to exercise async archiving');
|
||||
$oHostDbMaster->sqlExecute("create table wal_activity (id int)");
|
||||
|
||||
Reference in New Issue
Block a user