1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00

New config options compress-level and compress-level-network to control compression levels.

This commit is contained in:
David Steele 2015-03-02 20:36:12 -05:00
parent 49112eb6d3
commit 8e8f2f3c77
9 changed files with 267 additions and 58 deletions

View File

@ -379,6 +379,20 @@ Enable gzip compression. Files stored in the backup are compatible with command
default: y
example: compress=n
```
##### compress-level
Sets the zlib level (0-9) to be used for file compression when `compress=y`.
```
default: 6
example: compress-level=5
```
##### compress-level-network
Sets the zlib level (0-9) to be used for protocol compression when `compress=n` and the database is not on the same host as the backup. Protocol compression is used to reduce network traffic but can be disabled by setting `compress-level-network=0`. When `compress=y` the `compress-level-network` setting is ignored and `compress-level` is used instead so that the file is only compressed once. SSH compression is never enabled.
```
default: 3
example: compress-level-network=1
```
##### start-fast
Forces an immediate checkpoint (by passing true to the fast parameter of pg_start_backup()) so the backup begins immediately.
@ -434,6 +448,18 @@ When set then archive logs are not compressed immediately, but are instead compr
default: n
example: compress-async=y
```
##### compress
Overrides the setting in the `backup` section.
##### compress-level
Overrides the setting in the `backup` section.
##### compress-level-network
Overrides the setting in the `backup` section.
##### archive-max-mb
Limits the amount of archive log that will be written locally. After the limit is reached, the following will happen:
@ -488,7 +514,11 @@ example: archive-retention=2
### restore section
?????
##### compress-level-network
Overrides the setting in the `backup` section.
???????
### restore:option section

View File

@ -78,7 +78,8 @@ pg_backrest.pl [options] [operation]
####################################################################################################################################
# Global variables
####################################################################################################################################
my $oRemote; # Remote object
my $oRemote; # Remote protocol object
my $oLocal; # Local protocol object
my $strRemote; # Defines which side is remote, DB or BACKUP
####################################################################################################################################
@ -86,17 +87,41 @@ my $strRemote; # Defines which side is remote, DB or BACKUP
####################################################################################################################################
sub remote_get
{
if (!defined($oRemote) && $strRemote ne NONE)
my $bForceLocal = shift;
my $iCompressLevel = shift;
my $iCompressLevelNetwork = shift;
# Return the remote if is already defined
if (defined($oRemote))
{
return $oRemote;
}
# Return the remote when required
if ($strRemote ne NONE && !$bForceLocal)
{
$oRemote = new BackRest::Remote
(
config_key_load($strRemote eq DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST, true),
config_key_load($strRemote eq DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_USER, true),
config_key_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_REMOTE, true)
config_key_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_REMOTE, true),
undef, $iCompressLevel, $iCompressLevelNetwork
);
return $oRemote;
}
# Otherwise return local
if (!defined($oLocal))
{
$oLocal = new BackRest::Remote
(
undef, undef, undef,
undef, $iCompressLevel, $iCompressLevelNetwork
);
}
return $oRemote;
return $oLocal;
}
####################################################################################################################################
@ -221,7 +246,8 @@ if (operation_get() eq OP_ARCHIVE_PUSH)
param_get(PARAM_STANZA),
config_key_load($strSection, CONFIG_KEY_PATH, true),
$bArchiveLocal ? NONE : $strRemote,
$bArchiveLocal ? undef : remote_get()
remote_get($bArchiveLocal, config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL),
config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK))
);
# Init backup
@ -291,7 +317,8 @@ if (operation_get() eq OP_ARCHIVE_PUSH)
param_get(PARAM_STANZA),
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
$strRemote,
remote_get()
remote_get(false, config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL),
config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK))
);
# Init backup
@ -394,7 +421,9 @@ if (operation_get() eq OP_ARCHIVE_GET)
param_get(PARAM_STANZA),
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
$strRemote,
remote_get()
remote_get(false,
config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL),
config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK))
);
# Init the backup object
@ -419,7 +448,9 @@ my $oFile = new BackRest::File
param_get(PARAM_STANZA),
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
$strRemote,
remote_get()
remote_get(false,
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL),
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK))
);
####################################################################################################################################

View File

@ -54,7 +54,12 @@ sub param_get
log_level_set(OFF, OFF);
# Create the remote object
my $oRemote = new BackRest::Remote();
my $oRemote = new BackRest::Remote
(
undef, # Host
undef, # User
'remote' # Command
);
# Create the file object
my $oFile = new BackRest::File
@ -62,12 +67,9 @@ my $oFile = new BackRest::File
undef,
undef,
undef,
$oRemote
$oRemote,
);
# Write the greeting so remote process knows who we are
$oRemote->greeting_write();
# Command string
my $strCommand = OP_NOOP;

View File

@ -46,7 +46,7 @@ our @EXPORT = qw(config_load config_key_load config_section_load operation_get o
CONFIG_KEY_LEVEL_FILE CONFIG_KEY_LEVEL_CONSOLE
CONFIG_KEY_COMPRESS CONFIG_KEY_PSQL CONFIG_KEY_REMOTE
CONFIG_KEY_COMPRESS CONFIG_KEY_COMPRESS_LEVEL CONFIG_KEY_COMPRESS_LEVEL_NETWORK CONFIG_KEY_PSQL CONFIG_KEY_REMOTE
CONFIG_KEY_FULL_RETENTION CONFIG_KEY_DIFFERENTIAL_RETENTION CONFIG_KEY_ARCHIVE_RETENTION_TYPE
CONFIG_KEY_ARCHIVE_RETENTION
@ -160,6 +160,8 @@ use constant
CONFIG_KEY_LEVEL_CONSOLE => 'level-console',
CONFIG_KEY_COMPRESS => 'compress',
CONFIG_KEY_COMPRESS_LEVEL => 'compress-level',
CONFIG_KEY_COMPRESS_LEVEL_NETWORK => 'compress-level-network',
CONFIG_KEY_PSQL => 'psql',
CONFIG_KEY_REMOTE => 'remote',
@ -399,6 +401,61 @@ sub config_valid
}
}
# Default compression levels
if (!defined(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL)))
{
config_key_set(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL, 6);
}
else
{
if (config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL) < 0 ||
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL) > 9)
{
die &log(ERROR, 'compress-level must be between 0 and 9');
}
}
if (!defined(config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL)))
{
config_key_set(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL,
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL));
}
else
{
if (config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL) < 0 ||
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL) > 9)
{
die &log(ERROR, 'compress-level must be between 0 and 9');
}
}
if (!defined(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK)))
{
config_key_set(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK, 3);
}
else
{
if (config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK) < 0 ||
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK) > 9)
{
die &log(ERROR, 'compress-level must be between 0 and 9');
}
}
if (!defined(config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK)))
{
config_key_set(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK,
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK));
}
else
{
if (config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK) < 0 ||
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK) > 9)
{
die &log(ERROR, 'compress-level must be between 0 and 9');
}
}
# Default thread-max
if (!defined(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX)))
{

View File

@ -12,7 +12,7 @@ use Carp;
# Exports
####################################################################################################################################
use Exporter qw(import);
our @EXPORT = qw(ERROR_CHECKSUM ERROR_CONFIG ERROR_PARAM ERROR_POSTMASTER_RUNNING ERROR_RESTORE_PATH_NOT_EMPTY);
our @EXPORT = qw(ERROR_CHECKSUM ERROR_CONFIG ERROR_PARAM ERROR_POSTMASTER_RUNNING ERROR_RESTORE_PATH_NOT_EMPTY ERROR_PROTOCOL);
####################################################################################################################################
# Exception Codes
@ -23,7 +23,8 @@ use constant
ERROR_CONFIG => 101,
ERROR_PARAM => 102,
ERROR_RESTORE_PATH_NOT_EMPTY => 103,
ERROR_POSTMASTER_RUNNING => 104
ERROR_POSTMASTER_RUNNING => 104,
ERROR_PROTOCOL => 105
};
####################################################################################################################################

View File

@ -127,9 +127,10 @@ sub new
$self->{oRemote} = $oRemote;
$self->{iThreadIdx} = $iThreadIdx;
if (!defined($self->{strRemote}) || $self->{strRemote} eq NONE)
# Remote object must be set
if (!defined($self->{oRemote}))
{
$self->{oRemote} = new BackRest::Remote();
confess &log(ASSERT, 'oRemote must be defined');
}
# If remote is defined check parameters and open session
@ -141,12 +142,6 @@ sub new
confess &log(ASSERT, 'strRemote must be "' . DB . '" or "' . BACKUP .
"\", $self->{strRemote} was passed");
}
# Remote object must be set
if (!defined($self->{oRemote}))
{
confess &log(ASSERT, 'oRemote must be defined');
}
}
return $self;

View File

@ -38,7 +38,7 @@ use constant
####################################################################################################################################
use constant
{
DEFAULT_BLOCK_SIZE => 8192
DEFAULT_BLOCK_SIZE => 1048576
};
####################################################################################################################################
@ -46,11 +46,13 @@ use constant
####################################################################################################################################
sub new
{
my $class = shift; # Class name
my $strHost = shift; # Host to connect to for remote (optional as this can also be used on the remote)
my $strUser = shift; # User to connect to for remote (must be set if strHost is set)
my $strCommand = shift; # Command to execute on remote
my $iBlockSize = shift; # Optionally, set the block size (defaults to DEFAULT_BLOCK_SIZE)
my $class = shift; # Class name
my $strHost = shift; # Host to connect to for remote (optional as this can also be used on the remote)
my $strUser = shift; # User to connect to for remote (must be set if strHost is set)
my $strCommand = shift; # Command to execute on remote ('remote' if this is the remote)
my $iBlockSize = shift; # Optionally, set the block size (defaults to DEFAULT_BLOCK_SIZE)
my $iCompressLevel = shift; # Set compression level
my $iCompressLevelNetwork = shift; # Set compression level for network only compression
# Create the class hash
my $self = {};
@ -69,6 +71,10 @@ sub new
$self->{iBlockSize} = $iBlockSize;
}
# Set compress levels
$self->{iCompressLevel} = $iCompressLevel;
$self->{iCompressLevelNetwork} = $iCompressLevelNetwork;
# If host is defined then make a connnection
if (defined($strHost))
{
@ -104,6 +110,26 @@ sub new
($self->{hIn}, $self->{hOut}, $self->{hErr}, $self->{pId}) = $self->{oSSH}->open3($self->{strCommand});
$self->greeting_read();
$self->setting_write($self->{iBlockSize}, $self->{iCompressLevel}, $self->{iCompressLevelNetwork});
}
elsif (defined($strCommand) && $strCommand eq 'remote')
{
# Write the greeting so master process knows who we are
$self->greeting_write();
# Read settings from master
($self->{iBlockSize}, $self->{iCompressLevel}, $self->{iCompressLevelNetwork}) = $self->setting_read();
}
# Check compress levels
if (!defined($self->{iCompressLevel}))
{
confess &log(ASSERT, 'iCompressLevel must be set');
}
if (!defined($self->{iCompressLevelNetwork}))
{
confess &log(ASSERT, 'iCompressLevelNetwork must be set');
}
return $self;
@ -139,7 +165,9 @@ sub clone
$self->{strHost},
$self->{strUser},
$self->{strCommand},
$self->{iBlockSize}
$self->{iBlockSize},
$self->{iCompressLevel},
$self->{iCompressLevelNetwork}
);
}
@ -168,10 +196,50 @@ sub greeting_write
{
my $self = shift;
if (!syswrite(*STDOUT, "$self->{strGreeting}\n"))
$self->write_line(*STDOUT, $self->{strGreeting});
}
####################################################################################################################################
# SETTING_READ
#
# Read the settings from the master process.
####################################################################################################################################
sub setting_read
{
my $self = shift;
# Tokenize the settings
my @stryToken = split(/ /, $self->read_line(*STDIN));
# Make sure there are the correct number of tokens
if (@stryToken != 4)
{
confess 'unable to write greeting';
confess &log(ASSERT, 'settings token count is invalid', ERROR_PROTOCOL);
}
# Check for the setting token just to be sure
if ($stryToken[0] ne 'setting')
{
confess &log(ASSERT, 'settings token 0 must be \'setting\'');
}
# Return the settings
return $stryToken[1], $stryToken[2], $stryToken[3];
}
####################################################################################################################################
# SETTING_WRITE
#
# Send settings to the remote process.
####################################################################################################################################
sub setting_write
{
my $self = shift;
my $iBlockSize = shift; # Optionally, set the block size (defaults to DEFAULT_BLOCK_SIZE)
my $iCompressLevel = shift; # Set compression level
my $iCompressLevelNetwork = shift; # Set compression level for network only compression
$self->write_line($self->{hIn}, "setting ${iBlockSize} ${iCompressLevel} ${iCompressLevelNetwork}");
}
####################################################################################################################################
@ -306,13 +374,11 @@ sub write_line
my $hOut = shift;
my $strBuffer = shift;
$strBuffer = $strBuffer . "\n";
my $iLineOut = syswrite($hOut, $strBuffer . "\n");
my $iLineOut = syswrite($hOut, $strBuffer, length($strBuffer));
if (!defined($iLineOut) || $iLineOut != length($strBuffer))
if (!defined($iLineOut) || $iLineOut != length($strBuffer) + 1)
{
confess 'unable to write ' . length($strBuffer) . ' byte(s)';
confess &log(ERROR, "unable to write ${strBuffer}: $!", ERROR_PROTOCOL);
}
}
@ -707,7 +773,8 @@ sub binary_xfer
# Initialize inflate object and check for errors
my ($oZLib, $iZLibStatus) =
new Compress::Raw::Zlib::Deflate(WindowBits => 15 & $bDestinationCompress ? WANT_GZIP : 0,
Level => !$bSourceCompressed ? Z_BEST_SPEED : Z_DEFAULT_COMPRESSION,
Level => $bDestinationCompress ? $self->{iCompressLevel} :
$self->{iCompressLevelNetwork},
Bufsize => $self->{iBlockSize}, AppendOutput => 1);
if ($iZLibStatus != Z_OK)

View File

@ -1209,7 +1209,7 @@ sub BackRestTestBackup_Test
my $strArchiveChecksum = '1c7e00fd09b9dd11fc2966590b3e3274645dd031';
my $iArchiveMax = 3;
my $strXlogPath = BackRestTestCommon_DbCommonPathGet() . '/pg_xlog';
my $strArchiveTestFile = BackRestTestCommon_DataPathGet() . '/test.archive.bin';
my $strArchiveTestFile = BackRestTestCommon_DataPathGet() . '/test.archive2.bin';
# Print test banner
&log(INFO, 'BACKUP MODULE ******************************************************************');
@ -1221,7 +1221,20 @@ sub BackRestTestBackup_Test
(
$strHost,
$strUserBackRest,
BackRestTestCommon_CommandRemoteGet()
BackRestTestCommon_CommandRemoteGet(),
undef, # Buffer size
3, # Compress level
1, # Compress network level
);
my $oLocal = new BackRest::Remote
(
undef,
undef,
undef,
undef, # Buffer size
3, # Compress level
1, # Compress network level
);
#-------------------------------------------------------------------------------------------------------------------------------
@ -1257,7 +1270,7 @@ sub BackRestTestBackup_Test
$strStanza,
BackRestTestCommon_BackupPathGet(),
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
))->clone();
BackRestTestBackup_Create($bRemote, false);
@ -1375,7 +1388,7 @@ sub BackRestTestBackup_Test
$strStanza,
BackRestTestCommon_BackupPathGet(),
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
))->clone();
BackRestTestBackup_Create($bRemote, false);
@ -1513,7 +1526,7 @@ sub BackRestTestBackup_Test
$strStanza,
BackRestTestCommon_BackupPathGet(),
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
BackRestTestBackup_ManifestFileCreate(\%oManifest, 'base', 'PG_VERSION', '9.3',
@ -1789,7 +1802,7 @@ sub BackRestTestBackup_Test
$strStanza,
BackRestTestCommon_BackupPathGet(),
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
# Create the test directory

View File

@ -89,13 +89,26 @@ sub BackRestTestFile_Test
&log(INFO, 'FILE MODULE ********************************************************************');
#-------------------------------------------------------------------------------------------------------------------------------
# Create remote
# Create remotes
#-------------------------------------------------------------------------------------------------------------------------------
my $oRemote = new BackRest::Remote
(
$strHost,
$strUser,
BackRestTestCommon_CommandRemoteGet()
BackRestTestCommon_CommandRemoteGet(),
undef, # Buffer size
3, # Compress level
1, # Compress network level
);
my $oLocal = new BackRest::Remote
(
undef,
undef,
undef,
undef, # Buffer size
3, # Compress level
1, # Compress network level
);
#-------------------------------------------------------------------------------------------------------------------------------
@ -116,7 +129,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
# Loop through error
@ -224,7 +237,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
))->clone(1);
# Loop through source exists
@ -323,7 +336,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
# Loop through exists
@ -415,7 +428,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'db' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
my $lTimeBegin = gettimeofday();
@ -484,7 +497,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
for (my $bError = 0; $bError <= 1; $bError++)
@ -626,7 +639,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
for (my $bSort = false; $bSort <= true; $bSort++)
@ -752,7 +765,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
# Loop through exists
@ -855,7 +868,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
# Loop through error
@ -946,7 +959,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$bRemote ? 'backup' : undef,
$bRemote ? $oRemote : undef
$bRemote ? $oRemote : $oLocal
);
# Loop through exists
@ -1047,7 +1060,7 @@ sub BackRestTestFile_Test
$strStanza,
$strTestPath,
$strRemote,
defined($strRemote) ? $oRemote : undef
defined($strRemote) ? $oRemote : $oLocal
);
# Loop through source path types