1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
pgbackrest/lib/pgBackRest/Protocol/Helper.pm
David Steele 1b48684713 The archive-push command is implemented entirely in C.
This new implementation should behave exactly like the old Perl code with the exception of updated log messages.

Remove as much of the Perl code as possible without breaking other commands.
2019-03-29 13:26:33 +00:00

494 lines
20 KiB
Perl

####################################################################################################################################
# Create and manage protocol objects.
####################################################################################################################################
package pgBackRest::Protocol::Helper;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Remote::Master;
use pgBackRest::Version;
####################################################################################################################################
# Operation constants
####################################################################################################################################
# Backup module
use constant OP_BACKUP_FILE => 'backupFile';
push @EXPORT, qw(OP_BACKUP_FILE);
# Archive Module
use constant OP_ARCHIVE_GET_CHECK => 'archiveCheck';
push @EXPORT, qw(OP_ARCHIVE_GET_CHECK);
# Check Module
use constant OP_CHECK_BACKUP_INFO_CHECK => 'backupInfoCheck';
push @EXPORT, qw(OP_CHECK_BACKUP_INFO_CHECK);
# Db Module
use constant OP_DB_CONNECT => 'dbConnect';
push @EXPORT, qw(OP_DB_CONNECT);
use constant OP_DB_EXECUTE_SQL => 'dbExecSql';
push @EXPORT, qw(OP_DB_EXECUTE_SQL);
use constant OP_DB_INFO => 'dbInfo';
push @EXPORT, qw(OP_DB_INFO);
# Storage Module
use constant OP_STORAGE_OPEN_READ => 'storageOpenRead';
push @EXPORT, qw(OP_STORAGE_OPEN_READ);
use constant OP_STORAGE_OPEN_WRITE => 'storageOpenWrite';
push @EXPORT, qw(OP_STORAGE_OPEN_WRITE);
use constant OP_STORAGE_CIPHER_PASS_USER => 'storageCipherPassUser';
push @EXPORT, qw(OP_STORAGE_CIPHER_PASS_USER);
use constant OP_STORAGE_EXISTS => 'storageExists';
push @EXPORT, qw(OP_STORAGE_EXISTS);
use constant OP_STORAGE_HASH_SIZE => 'storageHashSize';
push @EXPORT, qw(OP_STORAGE_HASH_SIZE);
use constant OP_STORAGE_LIST => 'storageList';
push @EXPORT, qw(OP_STORAGE_LIST);
use constant OP_STORAGE_MANIFEST => 'storageManifest';
push @EXPORT, qw(OP_STORAGE_MANIFEST);
use constant OP_STORAGE_MOVE => 'storageMove';
push @EXPORT, qw(OP_STORAGE_MOVE);
use constant OP_STORAGE_PATH_GET => 'storagePathGet';
push @EXPORT, qw(OP_STORAGE_PATH_GET);
# Restore module
use constant OP_RESTORE_FILE => 'restoreFile';
push @EXPORT, qw(OP_RESTORE_FILE);
# Wait
use constant OP_WAIT => 'wait';
push @EXPORT, qw(OP_WAIT);
####################################################################################################################################
# Module variables
####################################################################################################################################
my $hProtocol = {}; # Global remote hash that is created on first request
####################################################################################################################################
# isRepoLocal
#
# Is the backup/archive repository local? This does not take into account the spool path.
####################################################################################################################################
sub isRepoLocal
{
# Not valid for remote
if (cfgCommandTest(CFGCMD_REMOTE) && !cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP))
{
confess &log(ASSERT, 'isRepoLocal() not valid on ' . cfgOption(CFGOPT_TYPE) . ' remote');
}
return cfgOptionTest(CFGOPT_REPO_HOST) ? false : true;
}
push @EXPORT, qw(isRepoLocal);
####################################################################################################################################
# isDbLocal - is the database local?
####################################################################################################################################
sub isDbLocal
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$iRemoteIdx,
) =
logDebugParam
(
__PACKAGE__ . '::isDbLocal', \@_,
{name => 'iRemoteIdx', optional => true, default => cfgOptionValid(CFGOPT_HOST_ID) ? cfgOption(CFGOPT_HOST_ID) : 1,
trace => true},
);
# Not valid for remote
if (cfgCommandTest(CFGCMD_REMOTE) && !cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_DB))
{
confess &log(ASSERT, 'isDbLocal() not valid on ' . cfgOption(CFGOPT_TYPE) . ' remote');
}
my $bLocal = cfgOptionTest(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iRemoteIdx)) ? false : true;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bLocal', value => $bLocal, trace => true}
);
}
push @EXPORT, qw(isDbLocal);
####################################################################################################################################
# Gets the parameters required to setup the protocol
####################################################################################################################################
sub protocolParam
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCommand,
$strRemoteType,
$iRemoteIdx,
$strBackRestBin,
$iProcessIdx,
) =
logDebugParam
(
__PACKAGE__ . '::protocolParam', \@_,
{name => 'strCommand'},
{name => 'strRemoteType'},
{name => 'iRemoteIdx', default => cfgOptionValid(CFGOPT_HOST_ID) ? cfgOption(CFGOPT_HOST_ID) : 1},
{name => 'strBackRestBin', optional => true},
{name => 'iProcessIdx', optional => true,
default => cfgOptionValid(CFGOPT_PROCESS) ? cfgOption(CFGOPT_PROCESS, false) : undef},
);
# Return the remote when required
my $iOptionIdCmd = CFGOPT_REPO_HOST_CMD;
my $iOptionIdConfig = CFGOPT_REPO_HOST_CONFIG;
my $iOptionIdConfigIncludePath = CFGOPT_REPO_HOST_CONFIG_INCLUDE_PATH;
my $iOptionIdConfigPath = CFGOPT_REPO_HOST_CONFIG_PATH;
my $iOptionIdHost = CFGOPT_REPO_HOST;
my $iOptionIdUser = CFGOPT_REPO_HOST_USER;
my $strOptionDbPath = undef;
my $strOptionDbPort = undef;
my $strOptionDbSocketPath = undef;
my $strOptionSshPort = CFGOPT_REPO_HOST_PORT;
if ($strRemoteType eq CFGOPTVAL_REMOTE_TYPE_DB)
{
$iOptionIdCmd = cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iRemoteIdx);
$iOptionIdConfig = cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iRemoteIdx);
$iOptionIdConfigIncludePath = cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH, $iRemoteIdx);
$iOptionIdConfigPath = cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_PATH, $iRemoteIdx);
$iOptionIdHost = cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iRemoteIdx);
$iOptionIdUser = cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iRemoteIdx);
$strOptionSshPort = cfgOptionIdFromIndex(CFGOPT_PG_HOST_PORT, $iRemoteIdx);
}
# Db path is not valid in all contexts (restore, for instance)
if (cfgOptionValid(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx)))
{
$strOptionDbPath =
cfgOptionSource(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx)) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx));
}
# Db port is not valid in all contexts (restore, for instance)
if (cfgOptionValid(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iRemoteIdx)))
{
$strOptionDbPort =
cfgOptionSource(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iRemoteIdx)) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iRemoteIdx));
}
# Db socket is not valid in all contexts (restore, for instance)
if (cfgOptionValid(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $iRemoteIdx)))
{
$strOptionDbSocketPath =
cfgOptionSource(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $iRemoteIdx)) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $iRemoteIdx));
}
# Build hash to set and override command options
my $rhCommandOption =
{
&CFGOPT_COMMAND => {value => $strCommand},
&CFGOPT_PROCESS => {value => defined($iProcessIdx) ? $iProcessIdx : 0},
&CFGOPT_CONFIG =>
{value => cfgOptionValid($iOptionIdConfig) && cfgOptionSource($iOptionIdConfig) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption($iOptionIdConfig)},
&CFGOPT_CONFIG_INCLUDE_PATH =>
{value => cfgOptionValid($iOptionIdConfigIncludePath) &&
cfgOptionSource($iOptionIdConfigIncludePath) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption($iOptionIdConfigIncludePath)},
&CFGOPT_CONFIG_PATH =>
{value => cfgOptionValid($iOptionIdConfigPath) && cfgOptionSource($iOptionIdConfigPath) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption($iOptionIdConfigPath)},
&CFGOPT_TYPE => {value => $strRemoteType},
&CFGOPT_LOG_PATH => {},
&CFGOPT_LOCK_PATH => {},
# Only enable file logging on the remote when requested
&CFGOPT_LOG_LEVEL_FILE => {value => cfgOption(CFGOPT_LOG_SUBPROCESS) ? cfgOption(CFGOPT_LOG_LEVEL_FILE) : lc(OFF)},
# Don't pass CFGOPT_LOG_LEVEL_STDERR because in the case of the local process calling the remote process the
# option will be set to 'protocol' which is not a valid value from the command line.
&CFGOPT_LOG_LEVEL_STDERR => {},
cfgOptionIdFromIndex(CFGOPT_PG_PATH, 1) => {value => $strOptionDbPath},
cfgOptionIdFromIndex(CFGOPT_PG_PORT, 1) => {value => $strOptionDbPort},
cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, 1) => {value => $strOptionDbSocketPath},
# Set protocol options explicitly so values are not picked up from remote config files
&CFGOPT_BUFFER_SIZE => {value => cfgOption(CFGOPT_BUFFER_SIZE)},
&CFGOPT_COMPRESS_LEVEL => {value => cfgOption(CFGOPT_COMPRESS_LEVEL)},
&CFGOPT_COMPRESS_LEVEL_NETWORK => {value => cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK)},
&CFGOPT_PROTOCOL_TIMEOUT => {value => cfgOption(CFGOPT_PROTOCOL_TIMEOUT)}
};
# Override some per-db options that shouldn't be passed to the command. ??? This could be done better as a new define
# for these options which would then be implemented in cfgCommandWrite().
for (my $iOptionIdx = 1; $iOptionIdx <= cfgOptionIndexTotal(CFGOPT_PG_HOST); $iOptionIdx++)
{
if ($iOptionIdx != 1)
{
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_PATH, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $iOptionIdx)} = {};
}
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_PORT, $iOptionIdx)} = {};
}
# Generate the remote command
my $strRemoteCommand = cfgCommandWrite(
CFGCMD_REMOTE, true, defined($strBackRestBin) ? $strBackRestBin : cfgOption($iOptionIdCmd), undef, $rhCommandOption);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strRemoteHost', value => cfgOption($iOptionIdHost)},
{name => 'strRemoteHostUser', value => cfgOption($iOptionIdUser)},
{name => 'strRemoteHostSshPort', value => cfgOption($strOptionSshPort, false)},
{name => 'strRemoteCommand', value => $strRemoteCommand},
);
}
####################################################################################################################################
# protocolGet
#
# Get the protocol object or create it if does not exist. Shared protocol objects are used because they create an SSH connection
# to the remote host and the number of these connections should be minimized.
####################################################################################################################################
sub protocolGet
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strRemoteType,
$iRemoteIdx,
$bCache,
$strBackRestBin,
$iProcessIdx,
$strCommand,
) =
logDebugParam
(
__PACKAGE__ . '::protocolGet', \@_,
{name => 'strRemoteType'},
{name => 'iRemoteIdx', default => cfgOptionValid(CFGOPT_HOST_ID) ? cfgOption(CFGOPT_HOST_ID) : 1},
{name => 'bCache', optional => true, default => true},
{name => 'strBackRestBin', optional => true},
{name => 'iProcessIdx', optional => true},
{name => 'strCommand', optional => true,
default => cfgOptionValid(CFGOPT_COMMAND) ? cfgOption(CFGOPT_COMMAND) : cfgCommandName(cfgCommandGet())},
);
# Protocol object
my $oProtocol;
# If no remote requested or if the requested remote type is local then return a local protocol object
if (!cfgOptionTest(
cfgOptionIdFromIndex($strRemoteType eq CFGOPTVAL_REMOTE_TYPE_DB ? CFGOPT_PG_HOST : CFGOPT_REPO_HOST, $iRemoteIdx)))
{
confess &log(ASSERT, 'protocol cannot be created when remote host is not specified');
}
# Else create the remote protocol
else
{
# Set protocol to cached value
$oProtocol =
$bCache && defined($$hProtocol{$strRemoteType}{$iRemoteIdx}) ? $$hProtocol{$strRemoteType}{$iRemoteIdx} : undef;
if ($bCache && $$hProtocol{$strRemoteType}{$iRemoteIdx})
{
$oProtocol = $$hProtocol{$strRemoteType}{$iRemoteIdx};
logDebugMisc($strOperation, 'found cached protocol');
# Issue a noop on protocol pulled from the cache to be sure it is still functioning. It's better to get an error at
# request time than somewhere randomly later.
$oProtocol->noOp();
}
# If protocol was not returned from cache then create it
if (!defined($oProtocol))
{
logDebugMisc($strOperation, 'create (' . ($bCache ? '' : 'un') . 'cached) remote protocol');
my ($strRemoteHost, $strRemoteHostUser, $strRemoteHostSshPort, $strRemoteCommand) = protocolParam(
$strCommand, $strRemoteType, $iRemoteIdx, {strBackRestBin => $strBackRestBin, iProcessIdx => $iProcessIdx});
$oProtocol = new pgBackRest::Protocol::Remote::Master
(
cfgOption(CFGOPT_CMD_SSH),
$strRemoteCommand,
cfgOption(CFGOPT_BUFFER_SIZE),
cfgOption(CFGOPT_COMPRESS_LEVEL),
cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK),
$strRemoteHost,
$strRemoteHostUser,
$strRemoteHostSshPort,
cfgOption(CFGOPT_PROTOCOL_TIMEOUT)
);
# Cache the protocol
if ($bCache)
{
$$hProtocol{$strRemoteType}{$iRemoteIdx} = $oProtocol;
}
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oProtocol', value => $oProtocol, trace => true}
);
}
push @EXPORT, qw(protocolGet);
####################################################################################################################################
# protocolList - list all active protocols
####################################################################################################################################
sub protocolList
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strRemoteType,
$iRemoteIdx,
) =
logDebugParam
(
__PACKAGE__ . '::protocolList', \@_,
{name => 'strRemoteType', required => false, trace => true},
{name => 'iRemoteIdx', required => false, trace => true},
);
my @oyProtocol;
if (!defined($strRemoteType))
{
foreach my $strRemoteType (sort(keys(%{$hProtocol})))
{
push(@oyProtocol, protocolList($strRemoteType));
}
}
elsif (!defined($iRemoteIdx))
{
foreach my $iRemoteIdx (sort(keys(%{$hProtocol->{$strRemoteType}})))
{
push(@oyProtocol, protocolList($strRemoteType, $iRemoteIdx));
}
}
elsif (defined($hProtocol->{$strRemoteType}{$iRemoteIdx}))
{
push(@oyProtocol, {strRemoteType => $strRemoteType, iRemoteIdx => $iRemoteIdx});
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oyProtocol', value => \@oyProtocol, trace => true}
);
}
####################################################################################################################################
# protocolDestroy
#
# Undefine the protocol if it is stored locally.
####################################################################################################################################
sub protocolDestroy
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strRemoteType,
$iRemoteIdx,
$bComplete,
) =
logDebugParam
(
__PACKAGE__ . '::protocolDestroy', \@_,
{name => 'strRemoteType', required => false},
{name => 'iRemoteIdx', required => false},
{name => 'bComplete', default => false},
);
my $iExitStatus = 0;
foreach my $rhProtocol (protocolList($strRemoteType, $iRemoteIdx))
{
logDebugMisc(
$strOperation, 'found cached protocol',
{name => 'strRemoteType', value => $rhProtocol->{strRemoteType}},
{name => 'iRemoteIdx', value => $rhProtocol->{iRemoteIdx}});
$iExitStatus = $hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}}->close($bComplete);
delete($hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}});
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iExitStatus', value => $iExitStatus}
);
}
push @EXPORT, qw(protocolDestroy);
####################################################################################################################################
# protocolKeepAlive - call keepAlive() on all protocols
####################################################################################################################################
sub protocolKeepAlive
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strRemoteType,
$iRemoteIdx,
) =
logDebugParam
(
__PACKAGE__ . '::protocolDestroy', \@_,
{name => 'strRemoteType', required => false, trace => true},
{name => 'iRemoteIdx', required => false, trace => true},
);
foreach my $rhProtocol (protocolList($strRemoteType, $iRemoteIdx))
{
$hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}}->keepAlive();
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push @EXPORT, qw(protocolKeepAlive);
1;