db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Added buffer-size param and added config validation improvements.

Browse Source
This commit is contained in:
David Steele 2015-03-02 22:58:32 -05:00
parent 8e8f2f3c77
commit 7dbb11e8b8
6 changed files with 183 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
INSTALL.md
View File

@ -347,6 +347,40 @@ Sets console log level.
default: error
example: level-file=info
```
#### general section
The `general` section defines settings that are shared between all operations.
##### buffer-size
Set the buffer size used for copy, compress, and uncompress functions. A maximum of 3 buffers will be in use at a time per thread. An additional maximum of 256K per thread may be used for zlib buffers.
```
default: 1048576
allowed: 4096 - 8388608
example: buffer-size=8192
```
##### compress-level
Sets the zlib level to be used for file compression when `compress=y`.
This setting can be overridden in the `backup` and `archive` sections.
```
default: 6
allowed: 0 - 9
example: compress-level=9
```
##### compress-level-network
Sets the zlib level to be used for protocol compression when `compress=n` and the database is not on the same host as the backup. Protocol compression is used to reduce network traffic but can be disabled by setting `compress-level-network=0`. When `compress=y` the `compress-level-network` setting is ignored and `compress-level` is used instead so that the file is only compressed once. SSH compression is always disabled.
This setting can be overridden in the `backup` and `archive`, and `restore` sections.
```
default: 3
allowed: 0 - 9
example: compress-level-network=1
```
#### backup section
The backup section defines settings related to backup and archiving.
@ -379,20 +413,6 @@ Enable gzip compression. Files stored in the backup are compatible with command
default: y
example: compress=n
```
##### compress-level
Sets the zlib level (0-9) to be used for file compression when `compress=y`.
```
default: 6
example: compress-level=5
```
##### compress-level-network
Sets the zlib level (0-9) to be used for protocol compression when `compress=n` and the database is not on the same host as the backup. Protocol compression is used to reduce network traffic but can be disabled by setting `compress-level-network=0`. When `compress=y` the `compress-level-network` setting is ignored and `compress-level` is used instead so that the file is only compressed once. SSH compression is never enabled.
```
default: 3
example: compress-level-network=1
```
##### start-fast
Forces an immediate checkpoint (by passing true to the fast parameter of pg_start_backup()) so the backup begins immediately.
@ -448,18 +468,6 @@ When set then archive logs are not compressed immediately, but are instead compr
default: n
example: compress-async=y
```
##### compress
Overrides the setting in the `backup` section.
##### compress-level
Overrides the setting in the `backup` section.
##### compress-level-network
Overrides the setting in the `backup` section.
##### archive-max-mb
Limits the amount of archive log that will be written locally. After the limit is reached, the following will happen:
@ -514,11 +522,7 @@ example: archive-retention=2
### restore section
##### compress-level-network
Overrides the setting in the `backup` section.
???????
?????
### restore:option section

6
bin/pg_backrest.pl
View File

@ -105,7 +105,8 @@ sub remote_get
config_key_load($strRemote eq DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST, true),
config_key_load($strRemote eq DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_USER, true),
config_key_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_REMOTE, true),
undef, $iCompressLevel, $iCompressLevelNetwork
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_BUFFER_SIZE, true),
$iCompressLevel, $iCompressLevelNetwork
);
return $oRemote;
@ -117,7 +118,8 @@ sub remote_get
$oLocal = new BackRest::Remote
(
undef, undef, undef,
undef, $iCompressLevel, $iCompressLevelNetwork
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_BUFFER_SIZE, true),
$iCompressLevel, $iCompressLevelNetwork
);
}

169
lib/BackRest/Config.pm
View File

@ -35,7 +35,7 @@ our @EXPORT = qw(config_load config_key_load config_section_load operation_get o
PARAM_VERSION PARAM_HELP PARAM_TEST PARAM_TEST_DELAY PARAM_TEST_NO_FORK
CONFIG_SECTION_COMMAND CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP
CONFIG_SECTION_COMMAND CONFIG_SECTION_GENERAL CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP
CONFIG_SECTION_RESTORE CONFIG_SECTION_RECOVERY CONFIG_SECTION_RECOVERY_OPTION CONFIG_SECTION_TABLESPACE_MAP
CONFIG_SECTION_ARCHIVE CONFIG_SECTION_RETENTION CONFIG_SECTION_STANZA
@ -46,13 +46,16 @@ our @EXPORT = qw(config_load config_key_load config_section_load operation_get o
CONFIG_KEY_LEVEL_FILE CONFIG_KEY_LEVEL_CONSOLE
CONFIG_KEY_COMPRESS CONFIG_KEY_COMPRESS_LEVEL CONFIG_KEY_COMPRESS_LEVEL_NETWORK CONFIG_KEY_PSQL CONFIG_KEY_REMOTE
CONFIG_KEY_BUFFER_SIZE CONFIG_KEY_COMPRESS CONFIG_KEY_COMPRESS_LEVEL CONFIG_KEY_COMPRESS_LEVEL_NETWORK
CONFIG_KEY_PSQL CONFIG_KEY_REMOTE
CONFIG_KEY_FULL_RETENTION CONFIG_KEY_DIFFERENTIAL_RETENTION CONFIG_KEY_ARCHIVE_RETENTION_TYPE
CONFIG_KEY_ARCHIVE_RETENTION
CONFIG_KEY_STANDBY_MODE CONFIG_KEY_PRIMARY_CONNINFO CONFIG_KEY_TRIGGER_FILE CONFIG_KEY_RESTORE_COMMAND
CONFIG_KEY_ARCHIVE_CLEANUP_COMMAND CONFIG_KEY_RECOVERY_END_COMMAND);
CONFIG_KEY_ARCHIVE_CLEANUP_COMMAND CONFIG_KEY_RECOVERY_END_COMMAND
CONFIG_DEFAULT_BUFFER_SIZE CONFIG_DEFAULT_COMPRESS_LEVEL CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK);
####################################################################################################################################
# File/path constants
@ -134,6 +137,7 @@ use constant
{
CONFIG_SECTION_COMMAND => 'command',
CONFIG_SECTION_COMMAND_OPTION => 'command:option',
CONFIG_SECTION_GENERAL => 'general',
CONFIG_SECTION_LOG => 'log',
CONFIG_SECTION_BACKUP => 'backup',
CONFIG_SECTION_RESTORE => 'restore',
@ -159,6 +163,7 @@ use constant
CONFIG_KEY_LEVEL_FILE => 'level-file',
CONFIG_KEY_LEVEL_CONSOLE => 'level-console',
CONFIG_KEY_BUFFER_SIZE => 'buffer-size',
CONFIG_KEY_COMPRESS => 'compress',
CONFIG_KEY_COMPRESS_LEVEL => 'compress-level',
CONFIG_KEY_COMPRESS_LEVEL_NETWORK => 'compress-level-network',
@ -178,6 +183,36 @@ use constant
CONFIG_KEY_RECOVERY_END_COMMAND => 'recovery-end-command'
};
####################################################################################################################################
# Configuration defaults
####################################################################################################################################
use constant
{
CONFIG_DEFAULT_BUFFER_SIZE => 1048576,
CONFIG_DEFAULT_BUFFER_SIZE_MIN => 4096,
CONFIG_DEFAULT_BUFFER_SIZE_MAX => 8388608,
CONFIG_DEFAULT_COMPRESS_LEVEL => 6,
CONFIG_DEFAULT_COMPRESS_LEVEL_MIN => 0,
CONFIG_DEFAULT_COMPRESS_LEVEL_MAX => 9,
CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK => 3,
CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK_MIN => 0,
CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK_MAX => 9,
CONFIG_DEFAULT_THREAD_MAX => 1,
CONFIG_DEFAULT_THREAD_MAX_MIN => 1,
CONFIG_DEFAULT_THREAD_MAX_MAX => 64
};
####################################################################################################################################
# Validation constants
####################################################################################################################################
use constant
{
VALID_RANGE => 'range'
};
####################################################################################################################################
# Global variables
####################################################################################################################################
@ -401,71 +436,87 @@ sub config_valid
}
}
# Default compression levels
if (!defined(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL)))
# Validate buffer_size
my @iyRange = [CONFIG_DEFAULT_BUFFER_SIZE_MIN, CONFIG_DEFAULT_BUFFER_SIZE_MAX];
config_key_valid(CONFIG_SECTION_GENERAL, CONFIG_KEY_BUFFER_SIZE, CONFIG_DEFAULT_BUFFER_SIZE, VALID_RANGE, @iyRange);
# Validate compress-level
@iyRange = [CONFIG_DEFAULT_COMPRESS_LEVEL_MIN, CONFIG_DEFAULT_COMPRESS_LEVEL_MAX];
config_key_valid(CONFIG_SECTION_GENERAL, CONFIG_KEY_COMPRESS_LEVEL,
CONFIG_DEFAULT_COMPRESS_LEVEL, VALID_RANGE, @iyRange);
config_key_valid(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL,
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_COMPRESS_LEVEL), VALID_RANGE, @iyRange);
config_key_valid(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL,
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_COMPRESS_LEVEL), VALID_RANGE, @iyRange);
# Validate compress-level-network
@iyRange = [CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK_MIN, CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK_MAX];
config_key_valid(CONFIG_SECTION_GENERAL, CONFIG_KEY_COMPRESS_LEVEL_NETWORK,
CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK, VALID_RANGE, @iyRange);
config_key_valid(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK,
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_COMPRESS_LEVEL_NETWORK), VALID_RANGE, @iyRange);
config_key_valid(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK,
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_COMPRESS_LEVEL_NETWORK), VALID_RANGE, @iyRange);
config_key_valid(CONFIG_SECTION_RESTORE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK,
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_COMPRESS_LEVEL_NETWORK), VALID_RANGE, @iyRange);
# Validate thread-max
@iyRange = [CONFIG_DEFAULT_THREAD_MAX_MIN, CONFIG_DEFAULT_THREAD_MAX_MAX];
config_key_valid(CONFIG_SECTION_GENERAL, CONFIG_KEY_THREAD_MAX,
CONFIG_DEFAULT_THREAD_MAX, VALID_RANGE, @iyRange);
config_key_valid(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX,
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_THREAD_MAX), VALID_RANGE, @iyRange);
config_key_valid(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_THREAD_MAX,
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_THREAD_MAX), VALID_RANGE, @iyRange);
config_key_valid(CONFIG_SECTION_RESTORE, CONFIG_KEY_THREAD_MAX,
config_key_load(CONFIG_SECTION_GENERAL, CONFIG_KEY_THREAD_MAX), VALID_RANGE, @iyRange);
}
####################################################################################################################################
# CONFIG_KEY_VALID
#
# Validate that the config matches the specified range, and default if undefined
####################################################################################################################################
sub config_key_valid
{
my $strSection = shift;
my $strKey = shift;
my $strDefault = shift;
my $strValidType = shift;
my $stryValidDataRef = shift;
# Get the key value or set it to a default
my $strValue = defined($oConfig{$strSection}{$strKey}) ? $oConfig{$strSection}{$strKey} : $strDefault;
# Validate the value
if ($strValidType eq VALID_RANGE)
{
config_key_set(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL, 6);
if (!defined($strValue) || $strValue < $$stryValidDataRef[0] || $strValue > $$stryValidDataRef[1])
{
confess &log(ERROR, "${strSection}::${strKey} is " . (defined($strValue) ? $strValue : 'not set') .
', but should be between ' . $$stryValidDataRef[0] . ' and ' . $$stryValidDataRef[1]);
}
}
else
{
if (config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL) < 0 ||
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL) > 9)
{
die &log(ERROR, 'compress-level must be between 0 and 9');
}
confess &log(ASSERT, "invalid validation type ${strValidType}");
}
if (!defined(config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL)))
{
config_key_set(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL,
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL));
}
else
{
if (config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL) < 0 ||
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL) > 9)
{
die &log(ERROR, 'compress-level must be between 0 and 9');
}
}
$oConfig{$strSection}{$strKey} = $strValue;
if (!defined(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK)))
{
config_key_set(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK, 3);
}
else
{
if (config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK) < 0 ||
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK) > 9)
{
die &log(ERROR, 'compress-level must be between 0 and 9');
}
}
# Also do validation for the stanza section
my $strStanza = param_get(PARAM_STANZA);
if (!defined(config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK)))
if (substr($strSection, 0, length($strStanza) + 1) ne "${strStanza}:")
{
config_key_set(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_COMPRESS_LEVEL_NETWORK,
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK));
}
else
{
if (config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK) < 0 ||
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS_LEVEL_NETWORK) > 9)
{
die &log(ERROR, 'compress-level must be between 0 and 9');
}
}
# Default thread-max
if (!defined(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX)))
{
config_key_set(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX, 1);
}
if (!defined(config_key_load(CONFIG_SECTION_RESTORE, CONFIG_KEY_THREAD_MAX)))
{
config_key_set(CONFIG_SECTION_RESTORE, CONFIG_KEY_THREAD_MAX,
config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX, true));
config_key_valid("${strStanza}:${strSection}", $strKey, $strValue, $strValidType, $stryValidDataRef);
}
}

25
lib/BackRest/Remote.pm
View File

@ -33,14 +33,6 @@ use constant
NONE => 'none'
};
####################################################################################################################################
# Remote xfer default block size constant
####################################################################################################################################
use constant
{
DEFAULT_BLOCK_SIZE => 1048576
};
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
@ -50,7 +42,7 @@ sub new
my $strHost = shift; # Host to connect to for remote (optional as this can also be used on the remote)
my $strUser = shift; # User to connect to for remote (must be set if strHost is set)
my $strCommand = shift; # Command to execute on remote ('remote' if this is the remote)
my $iBlockSize = shift; # Optionally, set the block size (defaults to DEFAULT_BLOCK_SIZE)
my $iBlockSize = shift; # Buffer size
my $iCompressLevel = shift; # Set compression level
my $iCompressLevelNetwork = shift; # Set compression level for network only compression
@ -62,14 +54,7 @@ sub new
$self->{strGreeting} = 'PG_BACKREST_REMOTE ' . version_get();
# Set default block size
if (!defined($iBlockSize))
{
$self->{iBlockSize} = DEFAULT_BLOCK_SIZE;
}
else
{
$self->{iBlockSize} = $iBlockSize;
}
$self->{iBlockSize} = $iBlockSize;
# Set compress levels
$self->{iCompressLevel} = $iCompressLevel;
@ -121,6 +106,12 @@ sub new
($self->{iBlockSize}, $self->{iCompressLevel}, $self->{iCompressLevelNetwork}) = $self->setting_read();
}
# Check block size
if (!defined($self->{iBlockSize}))
{
confess &log(ASSERT, 'iBlockSize must be set');
}
# Check compress levels
if (!defined($self->{iCompressLevel}))
{

24
test/lib/BackRestTest/BackupTest.pm
View File

@ -1219,22 +1219,22 @@ sub BackRestTestBackup_Test
#-------------------------------------------------------------------------------------------------------------------------------
my $oRemote = BackRest::Remote->new
(
$strHost,
$strUserBackRest,
BackRestTestCommon_CommandRemoteGet(),
undef, # Buffer size
3, # Compress level
1, # Compress network level
$strHost, # Host
$strUserBackRest, # User
BackRestTestCommon_CommandRemoteGet(), # Command
CONFIG_DEFAULT_BUFFER_SIZE, # Buffer size
CONFIG_DEFAULT_COMPRESS_LEVEL, # Compress level
CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
);
my $oLocal = new BackRest::Remote
(
undef,
undef,
undef,
undef, # Buffer size
3, # Compress level
1, # Compress network level
undef, # Host
undef, # User
undef, # Command
CONFIG_DEFAULT_BUFFER_SIZE, # Buffer size
CONFIG_DEFAULT_COMPRESS_LEVEL, # Compress level
CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
);
#-------------------------------------------------------------------------------------------------------------------------------

27
test/lib/BackRestTest/FileTest.pm
View File

@ -21,6 +21,7 @@ use POSIX qw(ceil);
use lib dirname($0) . '/../lib';
use BackRest::Utility;
use BackRest::Config;
use BackRest::File;
use BackRest::Remote;
@ -91,24 +92,24 @@ sub BackRestTestFile_Test
#-------------------------------------------------------------------------------------------------------------------------------
# Create remotes
#-------------------------------------------------------------------------------------------------------------------------------
my $oRemote = new BackRest::Remote
my $oRemote = BackRest::Remote->new
(
$strHost,
$strUser,
BackRestTestCommon_CommandRemoteGet(),
undef, # Buffer size
3, # Compress level
1, # Compress network level
$strHost, # Host
$strUser, # User
BackRestTestCommon_CommandRemoteGet(), # Command
CONFIG_DEFAULT_BUFFER_SIZE, # Buffer size
CONFIG_DEFAULT_COMPRESS_LEVEL, # Compress level
CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
);
my $oLocal = new BackRest::Remote
(
undef,
undef,
undef,
undef, # Buffer size
3, # Compress level
1, # Compress network level
undef, # Host
undef, # User
undef, # Command
CONFIG_DEFAULT_BUFFER_SIZE, # Buffer size
CONFIG_DEFAULT_COMPRESS_LEVEL, # Compress level
CONFIG_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
);
#-------------------------------------------------------------------------------------------------------------------------------