db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Add lz4 compression support.

Browse Source 
LZ4 compresses data faster than gzip but at a lower ratio.  This can be a good tradeoff in certain scenarios.

Note that setting compress-type=lz4 will make new backups and archive incompatible (unrestorable) with prior versions of pgBackRest.
This commit is contained in:
David Steele
2020-03-10 14:45:27 -04:00
parent cc9d7315db
commit c279a00279
38 changed files with 829 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/lib/pgBackRestTest/Common/ContainerTest.pm
+2 -2
View File
@@ -361,7 +361,7 @@ sub containerBuild
" yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" .
" perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" .
" gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" .
" zlib-devel libxml2-devel lz4-devel";
" zlib-devel libxml2-devel lz4-devel lz4";
if ($strOS eq VM_CO6)
{
@@ -388,7 +388,7 @@ sub containerBuild
}
else
{
$strScript .= ' libjson-pp-perl liblz4-dev';
$strScript .= ' libjson-pp-perl liblz4-dev liblz4-tool';
}
}
test/lib/pgBackRestTest/Common/JobTest.pm
+3 -1
View File
@@ -434,7 +434,8 @@ sub run
buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/test.c", $strTestC);
# Create build.auto.h
my $strBuildAutoH = "";
my $strBuildAutoH =
(vmWithLz4($self->{oTest}->{&TEST_VM}) ? '#define HAVE_LIBLZ4' : '') . "\n";
buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/" . BUILD_AUTO_H, $strBuildAutoH);
@@ -502,6 +503,7 @@ sub run
"HARNESSFLAGS=${strHarnessFlags}\n" .
"TESTFLAGS=${strTestFlags}\n" .
"LDFLAGS=-lcrypto -lssl -lxml2 -lz" .
(vmWithLz4($self->{oTest}->{&TEST_VM}) ? ' -llz4' : '') .
(vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .
(vmWithBackTrace($self->{oTest}->{&TEST_VM}) && $self->{bBackTrace} ? ' -lbacktrace' : '') .
"\n" .
test/lib/pgBackRestTest/Common/VmTest.pm
+15
View File
@@ -48,6 +48,8 @@ use constant VMDEF_LCOV_VERSION => 'lcov-ver
push @EXPORT, qw(VMDEF_LCOV_VERSION);
use constant VMDEF_WITH_BACKTRACE => 'with-backtrace';
push @EXPORT, qw(VMDEF_WITH_BACKTRACE);
use constant VMDEF_WITH_LZ4 => 'with-lz4';
push @EXPORT, qw(VMDEF_WITH_LZ4);
####################################################################################################################################
# Valid OS base List
@@ -302,6 +304,7 @@ my $oyVm =
&VM_IMAGE => 'i386/ubuntu:12.04',
&VM_ARCH => VM_ARCH_I386,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_WITH_LZ4 => false,
&VM_DB =>
[
@@ -591,6 +594,18 @@ sub vmWithBackTrace
push @EXPORT, qw(vmWithBackTrace);
####################################################################################################################################
# Does the VM support liblz4?
####################################################################################################################################
sub vmWithLz4
{
my $strVm = shift;
return (defined($oyVm->{$strVm}{&VMDEF_WITH_LZ4}) ? $oyVm->{$strVm}{&VMDEF_WITH_LZ4} : true);
}
push @EXPORT, qw(vmWithLz4);
####################################################################################################################################
# Will integration tests be run in debug mode?
####################################################################################################################################
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
+2
View File
@@ -91,6 +91,8 @@ use constant NONE => 'none';
push @EXPORT, qw(NONE);
use constant GZ => 'gz';
push @EXPORT, qw(GZ);
use constant LZ4 => 'lz4';
push @EXPORT, qw(LZ4);
####################################################################################################################################
# new
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
+3 -3
View File
@@ -60,13 +60,13 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => GZ},
{vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => LZ4},
{vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => GZ},
{vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => GZ},
{vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => GZ},
{vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => GZ},
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => GZ},
{vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => LZ4},
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => LZ4},
{vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => GZ},
)
{
test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm
+2 -2
View File
@@ -43,12 +43,12 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ, error => 0},
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4, error => 0},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ, error => 1},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => NONE, error => 0},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ, error => 0},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => GZ, error => 1},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => LZ4, error => 1},
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => GZ, error => 0},
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => NONE, error => 0},
)
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
+3 -3
View File
@@ -83,14 +83,14 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ},
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => GZ},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => LZ4},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => GZ},
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => GZ},
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => GZ},
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => LZ4},
)
{
# Only run tests for this vm
test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm
+3 -3
View File
@@ -41,13 +41,13 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => true, compress => GZ},
{vm => VM1, remote => false, s3 => false, encrypt => true, compress => LZ4},
{vm => VM1, remote => true, s3 => true, encrypt => false, compress => GZ},
{vm => VM2, remote => false, s3 => true, encrypt => true, compress => GZ},
{vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => false, compress => GZ},
{vm => VM3, remote => true, s3 => true, encrypt => true, compress => GZ},
{vm => VM4, remote => false, s3 => true, encrypt => false, compress => GZ},
{vm => VM3, remote => true, s3 => true, encrypt => true, compress => LZ4},
{vm => VM4, remote => false, s3 => true, encrypt => false, compress => LZ4},
{vm => VM4, remote => true, s3 => false, encrypt => true, compress => GZ},
)
{
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
+10 -1
View File
@@ -49,6 +49,9 @@ sub run
{
my $self = shift;
# Should the test use lz4 compression?
my $bLz4Compress = true;
foreach my $bS3 (false, true)
{
foreach my $bHostBackup ($bS3 ? (true) : (false, true))
@@ -60,9 +63,15 @@ sub run
foreach my $strBackupDestination (
$bS3 || $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER))
{
my $strCompressType = $bHostBackup && !$bHostStandby ? GZ : NONE;
my $strCompressType = $bHostBackup && !$bHostStandby ? (vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : GZ) : NONE;
my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false;
# If compression was used then switch it for the next test that uses compression
if ($strCompressType ne NONE)
{
$bLz4Compress = !$bLz4Compress;
}
# Increment the run, log, and decide whether this unit test should be run
my $hyVm = vmGet();
my $strDbVersionMostRecent = ${$hyVm->{$self->vm()}{&VM_DB_TEST}}[-1];