db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-06-14 23:44:58 +02:00
Code Issues Releases Activity

pgBackRest is now pure C.

Browse Source 
Remove embedded Perl from the distributed binary.  This includes code, configure, Makefile, and packages.  The distributed binary is now pure C.

Remove storagePathEnforceSet() from the C Storage object which allowed Perl to write outside of the storage base directory.  Update mock/all and real/all integration tests to use storageLocal() where they were violating this rule.

Remove "c" option that allowed the remote to tell if it was being called from C or Perl.

Code to convert options to JSON for passing to Perl (perl/config.c) has been moved to LibC since it is still required for Perl integration tests.

Update build and installation instructions in the user guide.

Remove all Perl unit tests.

Remove obsolete Perl code.  In particular this included all the Perl protocol code which required modifications to the Perl storage, manifest, and db objects that are still required for integration testing but only run locally.  Any remaining Perl code is required for testing, documentation, or code generation.

Rename perlReq to binReq in define.yaml to indicate that the binary is required for a test.  This had been the actual meaning for quite some time but the key was never renamed.
This commit is contained in:
David Steele
2019-12-13 17:55:41 -05:00
parent 1f2ce45e6b
commit f0ef73db70
80 changed files with 239 additions and 22415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build/lib/pgBackRestBuild/Config/BuildDefine.pm
View File

@ -134,7 +134,7 @@ sub helpFormatText
$strText .= "\n";
}
# Escape perl special characters
# Escape special characters
$strLine =~ s/\"/\\"/g;
my $strPart;

13
build/lib/pgBackRestBuild/Config/Data.pm
View File

@ -151,8 +151,6 @@ use constant CFGOPT_OUTPUT => 'output';
# Command-line only local/remote options
#-----------------------------------------------------------------------------------------------------------------------------------
use constant CFGOPT_C => 'c';
push @EXPORT, qw(CFGOPT_C);
use constant CFGOPT_COMMAND => 'command';
push @EXPORT, qw(CFGOPT_COMMAND);
use constant CFGOPT_PROCESS => 'process';
@ -1038,17 +1036,6 @@ my %hConfigDefine =
# Command-line only local/remote options
#-------------------------------------------------------------------------------------------------------------------------------
&CFGOPT_C =>
{
&CFGDEF_TYPE => CFGDEF_TYPE_BOOLEAN,
&CFGDEF_INTERNAL => true,
&CFGDEF_DEFAULT => false,
&CFGDEF_COMMAND =>
{
&CFGCMD_REMOTE => {},
}
},
&CFGOPT_COMMAND =>
{
&CFGDEF_TYPE => CFGDEF_TYPE_STRING,

128
build/lib/pgBackRestBuild/Embed/Build.pm
View File

@ -1,128 +0,0 @@
####################################################################################################################################
# Auto-Generate Embedded Perl Modules
####################################################################################################################################
package pgBackRestBuild::Embed::Build;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use pgBackRestBuild::Build::Common;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
####################################################################################################################################
# Constants
####################################################################################################################################
use constant BLDLCL_FILE_DEFINE => 'embed';
use constant BLDLCL_DATA_EMBED => '01-dataEmbed';
####################################################################################################################################
# Definitions for constants and data to build
####################################################################################################################################
my $strSummary = 'Embed Perl modules';
my $rhBuild =
{
&BLD_FILE =>
{
&BLDLCL_FILE_DEFINE =>
{
&BLD_SUMMARY => $strSummary,
&BLD_DATA =>
{
&BLDLCL_DATA_EMBED =>
{
&BLD_SUMMARY => 'Embedded Perl modules',
},
},
},
},
};
####################################################################################################################################
# Embed Perl modules
####################################################################################################################################
sub buildEmbed
{
my $oStorage = shift;
# Output embedded modules array
my $strBuildSource =
"static const EmbeddedModule embeddedModule[] =\n" .
"{\n";
foreach my $strModule (sort(keys(%{$oStorage->manifest('lib')})))
{
# Ignore non-Perl modules
if ($strModule =~ /\.pm$/)
{
# Load data
my $strData = trim(${$oStorage->get("lib/${strModule}")});
# Escape \ and quotes
$strData =~ s/\\/\\\\/g;
$strData =~ s/\"/\\\"/g;
$strBuildSource .=
" {\n" .
" .name = \"${strModule}\",\n" .
" .data =\n";
my $strLfLine = undef;
# Process each line
foreach my $strLine (split("\n", $strData))
{
# Remove comments
$strLine =~ s/^\s*\#.*$//;
$strLine =~ s/\#[^\'\"]*$//;
# Remove spacing in constant declarations
if ($strLine =~ /use constant [A-Z0-9_]+/)
{
$strLine =~ s/\s{2,}\=\> / \=\> /g;
}
# Remove leading/trailing spaces
$strLine = trim($strLine);
# If empty line then add it to the total empty lines
if ($strLine eq "")
{
$strLfLine .= "\\n";
}
# Output line (and empty lines if any)
else
{
$strBuildSource .=
(defined($strLfLine) ? " \"$strLfLine\"\n" : '') .
" \"$strLine\\n\"\n";
$strLfLine = undef;
}
}
$strBuildSource .=
" },\n";
}
}
$strBuildSource .=
"};";
$rhBuild->{&BLD_FILE}{&BLDLCL_FILE_DEFINE}{&BLD_DATA}{&BLDLCL_DATA_EMBED}{&BLD_SOURCE} = $strBuildSource;
return $rhBuild;
}
push @EXPORT, qw(buildEmbed);
1;

6
doc/xml/release.xml
View File

@ -14,6 +14,12 @@
<release-list>
<release date="XXXX-XX-XX" version="2.21dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-feature-list>
<release-item>
<p><backrest/> is now pure <proper>C</proper>.</p>
</release-item>
</release-feature-list>
<release-improvement-list>
<release-item>
<release-item-contributor-list>

79
doc/xml/user-guide.xml
View File

@ -83,13 +83,10 @@
<variable key="pg-group">postgres</variable>
<variable key="perl-lib-path">/usr/share/perl5</variable>
<variable key="perl-bin-path">/usr/bin</variable>
<variable key="backrest-repo-path">/var/lib/pgbackrest</variable>
<variable key="backrest-repo-cipher-type">aes-256-cbc</variable>
<variable key="backrest-repo-cipher-pass">zWaf6XtpjIVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO</variable>
<variable key="br-bin">{[perl-bin-path]}/pgbackrest</variable>
<variable key="br-bin">/usr/bin/pgbackrest</variable>
<variable key="br-user">pgbackrest</variable>
<variable key="br-group">{[br-user]}</variable>
<variable key="br-home-path">/home/{[br-user]}</variable>
@ -448,6 +445,23 @@
<block-define if="'{[package]}' eq 'none'" id="br-install">
<p><backrest/> needs to be installed from a package or installed manually as shown here.</p>
<execute-list host="{[host-build]}">
<title>Install dependencies</title>
<execute if="{[os-type-is-debian]}" user="root" pre="y">
<exe-cmd>
apt-get install postgresql-client libxml2</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
<execute if="{[os-type-is-centos]}" user="root" pre="y">
<exe-cmd>
yum install postgresql-libs
</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
</execute-list>
<execute-list host="{[br-install-host]}">
<title>Copy <backrest/> binary from build host</title>
@ -461,30 +475,7 @@
</execute>
</execute-list>
<!-- Perl installation -->
<p><backrest/> contains embedded Perl which requires some additional packages.</p>
<execute-list host="{[br-install-host]}">
<title>Install required Perl packages</title>
<execute if="{[os-type-is-debian]}" user="root" pre="y">
<exe-cmd>apt-get install perl</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
<execute if="{[os-type-is-centos6]}" user="root" pre="y">
<exe-cmd>yum install perl perl-Time-HiRes perl-parent perl-JSON
perl-Digest-SHA</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
<execute if="{[os-type-is-centos7]}" user="root" pre="y">
<exe-cmd>yum install perl perl-Time-HiRes perl-Digest-SHA perl-JSON-PP</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
</execute-list>
<p>Finally, <backrest/> requires log and configuration directories and a configuration file.</p>
<p><backrest/> requires log and configuration directories and a configuration file.</p>
<execute-list host="{[br-install-host]}">
<title>Create <backrest/> configuration file and directories</title>
@ -638,7 +629,7 @@
<p>This user guide is intended to be followed sequentially from beginning to end &amp;mdash; each section depends on the last. For example, the <link section="/backup">Backup</link> section relies on setup that is performed in the <link section="/quickstart">Quick Start</link> section. Once <backrest/> is up and running then skipping around is possible but following the user guide in order is recommended the first time through.</p>
<p>Although the examples are targeted at {[user-guide-os]} and <postgres/> {[pg-version]}, it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version. The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters. The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
<p>Although the examples are targeted at {[user-guide-os]} and <postgres/> {[pg-version]}, it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version. The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters. The <backrest/> commands will be the same on any Unix system though the location to install the executable may vary.
Configuration information and documentation for PostgreSQL can be found in the <postgres/> <link url='http://www.postgresql.org/docs/{[pg-version]}/static/index.html'>Manual</link>.</p>
@ -766,42 +757,20 @@
<execute if="{[os-type-is-debian]}" user="root" pre="y">
<exe-cmd>
apt-get install build-essential libssl-dev libxml2-dev libperl-dev zlib1g-dev
libpq-dev</exe-cmd>
apt-get install make gcc libpq-dev libssl-dev libxml2-dev</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
<execute if="{[os-type-is-centos6]}" user="root" pre="y">
<execute if="{[os-type-is-centos]}" user="root" pre="y">
<exe-cmd>
yum install build-essential gcc openssl-devel libxml2-devel
postgresql-devel perl-ExtUtils-Embed
</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
<execute if="{[os-type-is-centos7]}" user="root" pre="y">
<exe-cmd>
yum install build-essential gcc make openssl-devel libxml2-devel
postgresql-devel perl-ExtUtils-Embed
yum install make gcc postgresql-devel openssl-devel libxml2-devel
</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
</execute-list>
<p><backrest/> supports 32-bit distributions that build Perl with 64-bit integer support.</p>
<execute-list host="{[host-build]}">
<title>Check for 64-bit integers</title>
<execute output="y">
<exe-cmd>perl -V | grep USE_64_BIT_INT</exe-cmd>
</execute>
</execute-list>
<p>The <backrest/> executable is written in C. This allows certain time-critical commands (like async <cmd>archive-push</cmd>/<cmd>archive-get</cmd>) to run more quickly.</p>
<execute-list host="{[host-build]}">
<title>Build <backrest/> package</title>
<title>Compile <backrest/></title>
<execute>
<exe-cmd>cd /build/pgbackrest-release-{[version]}/src &amp;&amp; ./configure</exe-cmd>

1
lib/pgBackRest/Backup/Info.pm
View File

@ -22,7 +22,6 @@ use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;

48
lib/pgBackRest/Common/Cipher.pm
View File

@ -1,48 +0,0 @@
####################################################################################################################################
# Cipher Miscellaneous
####################################################################################################################################
package pgBackRest::Common::Cipher;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::LibC qw(:random :encode);
####################################################################################################################################
# cipherPassGen - generate a passphrase of the specified size (in bytes)
####################################################################################################################################
sub cipherPassGen
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$iKeySizeInBytes,
) =
logDebugParam
(
__PACKAGE__ . '::cipherPassGen', \@_,
{name => 'iKeySizeInBytes', default => 48},
);
# Create and base64 encode the key
my $strCipherPass = encodeToStr(ENCODE_TYPE_BASE64, cryptoRandomBytes($iKeySizeInBytes));
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strCipherPass', value => $strCipherPass, redact => true}
);
}
push @EXPORT, qw(cipherPassGen);
1;

187
lib/pgBackRest/Common/Io/Process.pm
View File

@ -1,187 +0,0 @@
####################################################################################################################################
# Process Execution, Management, and IO
####################################################################################################################################
package pgBackRest::Common::Io::Process;
use parent 'pgBackRest::Common::Io::Filter';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use IPC::Open3 qw(open3);
use POSIX qw(:sys_wait_h);
use Symbol 'gensym';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
####################################################################################################################################
# Package name constant
####################################################################################################################################
use constant COMMON_IO_PROCESS => __PACKAGE__;
push @EXPORT, qw(COMMON_IO_PROCESS);
####################################################################################################################################
# Amount of time to attempt to retrieve errors when a process terminates unexpectedly
####################################################################################################################################
use constant IO_ERROR_TIMEOUT => 5;
####################################################################################################################################
# new - use open3 to run the command and get the io handles
####################################################################################################################################
sub new
{
my $class = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oParent,
$strCommand,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oParent', trace => true},
{name => 'strCommand', trace => true},
);
# Bless with new class
my $self = $class->SUPER::new($oParent);
bless $self, $class;
# Use open3 to run the command
my ($iProcessId, $fhRead, $fhWrite, $fhReadError);
$fhReadError = gensym;
$iProcessId = IPC::Open3::open3($fhWrite, $fhRead, $fhReadError, $strCommand);
# Set handles
$self->handleReadSet($fhRead);
$self->handleWriteSet($fhWrite);
# Set variables
$self->{iProcessId} = $iProcessId;
$self->{fhReadError} = $fhReadError;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# error - handle errors
####################################################################################################################################
sub error
{
my $self = shift;
my $iCode = shift;
my $strMessage = shift;
my $strDetail = shift;
my $bClose = shift;
if (defined($self->{iProcessId}))
{
my $oWait = waitInit(defined($iCode) ? IO_ERROR_TIMEOUT : 0);
do
{
# Check the result
my $iResult = waitpid($self->{iProcessId}, $bClose ? 0 : WNOHANG);
# Error if the process exited unexpectedly
if ($iResult != 0)
{
# Get the exit status
my $iExitStatus = $iResult == -1 ? 255 : ${^CHILD_ERROR_NATIVE} >> 8;
# Drain the stderr stream
my $strError;
my $oIoError = new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle($self->id(), $self->{fhReadError}), 5, $self->bufferMax());
while (defined(my $strLine = $oIoError->readLine(true, false)))
{
$strError .= (defined($strError) ? "\n" : '') . $strLine;
}
delete($self->{iProcessId});
if (!$bClose || $iExitStatus != 0 || defined($strError))
{
my $iErrorCode =
$iExitStatus >= ERROR_MINIMUM && $iExitStatus <= ERROR_MAXIMUM ? $iExitStatus : ERROR_FILE_READ;
logErrorResult(
$iErrorCode, $self->id() . ' terminated unexpectedly' .
($iExitStatus != 255 ? sprintf(' [%03d]', $iExitStatus) : ''),
$strError);
}
}
}
while (waitMore($oWait));
if (defined($iCode))
{
$self->parent()->error($iCode, $strMessage, $strDetail);
}
}
else
{
confess &log(ASSERT, 'cannot call error() after process has been closed');
}
}
####################################################################################################################################
# Get process id
####################################################################################################################################
sub processId
{
my $self = shift;
return $self->{iProcessId};
}
####################################################################################################################################
# writeLine - check for error before writing line
####################################################################################################################################
sub writeLine
{
my $self = shift;
my $strBuffer = shift;
# Check if the process has exited abnormally (doesn't seem like we should need this, but the next syswrite does a hard
# abort if the remote process has already closed)
$self->error();
return $self->parent()->writeLine($strBuffer);
}
####################################################################################################################################
# close - check if the process terminated on error
####################################################################################################################################
sub close
{
my $self = shift;
if (defined($self->{iProcessId}))
{
$self->error(undef, undef, undef, true);
# Class parent close
$self->parent()->close();
}
return true;
}
1;

198
lib/pgBackRest/Db.pm
View File

@ -21,7 +21,6 @@ use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
@ -98,11 +97,6 @@ sub new
if (defined($self->{iRemoteIdx}))
{
$self->{strDbPath} = cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $self->{iRemoteIdx}));
if (!isDbLocal({iRemoteIdx => $self->{iRemoteIdx}}))
{
$self->{oProtocol} = protocolGet(CFGOPTVAL_REMOTE_TYPE_DB, $self->{iRemoteIdx});
}
}
# Return from function and log return values if any
@ -155,56 +149,46 @@ sub connect
# Only connect if not already connected
my $bResult = true;
# Run remotely
if (defined($self->{oProtocol}))
if (!defined($self->{oDb}))
{
# Set bResult to false if undef is returned
$bResult = $self->{oProtocol}->cmdExecute(OP_DB_CONNECT, undef, false, $bWarnOnError) ? true : false;
}
# Else run locally
else
{
if (!defined($self->{oDb}))
$self->{oDb} = new pgBackRest::LibC::PgClient(
cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $self->{iRemoteIdx}), false),
cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $self->{iRemoteIdx})), 'postgres',
cfgOption(CFGOPT_DB_TIMEOUT) * 1000);
if ($bWarnOnError)
{
$self->{oDb} = new pgBackRest::LibC::PgClient(
cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $self->{iRemoteIdx}), false),
cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $self->{iRemoteIdx})), 'postgres',
cfgOption(CFGOPT_DB_TIMEOUT) * 1000);
if ($bWarnOnError)
{
eval
{
$self->{oDb}->open();
return true;
}
or do
{
&log(WARN, exceptionMessage($EVAL_ERROR));
$bResult = false;
undef($self->{oDb});
}
}
else
eval
{
$self->{oDb}->open();
return true;
}
if (defined($self->{oDb}))
or do
{
my ($fDbVersion) = $self->versionGet();
&log(WARN, exceptionMessage($EVAL_ERROR));
$bResult = false;
if ($fDbVersion >= PG_VERSION_APPLICATION_NAME)
{
# Set application name for monitoring and debugging
$self->{oDb}->query(
"set application_name = '" . PROJECT_NAME . ' [' .
(cfgOptionValid(CFGOPT_COMMAND) ? cfgOption(CFGOPT_COMMAND) : cfgCommandName(cfgCommandGet())) . "]'");
undef($self->{oDb});
}
}
else
{
$self->{oDb}->open();
}
# Clear search path to prevent possible function overrides
$self->{oDb}->query("set search_path = 'pg_catalog'");
}
if (defined($self->{oDb}))
{
my ($fDbVersion) = $self->versionGet();
if ($fDbVersion >= PG_VERSION_APPLICATION_NAME)
{
# Set application name for monitoring and debugging
$self->{oDb}->query(
"set application_name = '" . PROJECT_NAME . ' [' .
(cfgOptionValid(CFGOPT_COMMAND) ? cfgOption(CFGOPT_COMMAND) : cfgCommandName(cfgCommandGet())) . "]'");
# Clear search path to prevent possible function overrides
$self->{oDb}->query("set search_path = 'pg_catalog'");
}
}
}
@ -243,22 +227,12 @@ sub executeSql
# Get the user-defined command for psql
my @stryResult;
# Run remotely
if (defined($self->{oProtocol}))
{
# Execute the command
@stryResult = @{$self->{oProtocol}->cmdExecute(OP_DB_EXECUTE_SQL, [$strSql, $bIgnoreError, $bResult], $bResult)};
}
# Else run locally
else
{
$self->connect();
my $strResult = $self->{oDb}->query($strSql);
$self->connect();
my $strResult = $self->{oDb}->query($strSql);
if (defined($strResult))
{
@stryResult = @{JSON::PP->new()->allow_nonref()->decode($strResult)};
}
if (defined($strResult))
{
@stryResult = @{JSON::PP->new()->allow_nonref()->decode($strResult)};
}
# Return from function and log return values if any
@ -346,63 +320,49 @@ sub info
#-------------------------------------------------------------------------------------------------------------------------------
if (!defined($self->{info}{$strDbPath}))
{
# Get info from remote
#---------------------------------------------------------------------------------------------------------------------------
if (defined($self->{oProtocol}))
# Open the control file and read system id and versions
#-----------------------------------------------------------------------------------------------------------------------
my $strControlFile = "${strDbPath}/" . DB_FILE_PGCONTROL;
my $hFile;
my $tBlock;
sysopen($hFile, $strControlFile, O_RDONLY)
or confess &log(ERROR, "unable to open ${strControlFile}", ERROR_FILE_OPEN);
# Read system identifier
sysread($hFile, $tBlock, 8) == 8
or confess &log(ERROR, "unable to read database system identifier");
$self->{info}{$strDbPath}{ullDbSysId} = unpack('Q', $tBlock);
# Read control version
sysread($hFile, $tBlock, 4) == 4
or confess &log(ERROR, "unable to read control version");
$self->{info}{$strDbPath}{iDbControlVersion} = unpack('L', $tBlock);
# Read catalog version
sysread($hFile, $tBlock, 4) == 4
or confess &log(ERROR, "unable to read catalog version");
$self->{info}{$strDbPath}{iDbCatalogVersion} = unpack('L', $tBlock);
# Close the control file
close($hFile);
# Get PostgreSQL version
$self->{info}{$strDbPath}{strDbVersion} =
$oPgControlVersionHash->{$self->{info}{$strDbPath}{iDbControlVersion}}
{$self->{info}{$strDbPath}{iDbCatalogVersion}};
if (!defined($self->{info}{$strDbPath}{strDbVersion}))
{
# Execute the command
($self->{info}{$strDbPath}{strDbVersion}, $self->{info}{$strDbPath}{iDbControlVersion},
$self->{info}{$strDbPath}{iDbCatalogVersion}, $self->{info}{$strDbPath}{ullDbSysId}) =
$self->{oProtocol}->cmdExecute(OP_DB_INFO, [$strDbPath], true);
}
# Get info locally
#---------------------------------------------------------------------------------------------------------------------------
else
{
# Open the control file and read system id and versions
#-----------------------------------------------------------------------------------------------------------------------
my $strControlFile = "${strDbPath}/" . DB_FILE_PGCONTROL;
my $hFile;
my $tBlock;
sysopen($hFile, $strControlFile, O_RDONLY)
or confess &log(ERROR, "unable to open ${strControlFile}", ERROR_FILE_OPEN);
# Read system identifier
sysread($hFile, $tBlock, 8) == 8
or confess &log(ERROR, "unable to read database system identifier");
$self->{info}{$strDbPath}{ullDbSysId} = unpack('Q', $tBlock);
# Read control version
sysread($hFile, $tBlock, 4) == 4
or confess &log(ERROR, "unable to read control version");
$self->{info}{$strDbPath}{iDbControlVersion} = unpack('L', $tBlock);
# Read catalog version
sysread($hFile, $tBlock, 4) == 4
or confess &log(ERROR, "unable to read catalog version");
$self->{info}{$strDbPath}{iDbCatalogVersion} = unpack('L', $tBlock);
# Close the control file
close($hFile);
# Get PostgreSQL version
$self->{info}{$strDbPath}{strDbVersion} =
$oPgControlVersionHash->{$self->{info}{$strDbPath}{iDbControlVersion}}
{$self->{info}{$strDbPath}{iDbCatalogVersion}};
if (!defined($self->{info}{$strDbPath}{strDbVersion}))
{
confess &log(
ERROR,
'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} .
' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" .
'HINT: is this version of PostgreSQL supported?',
ERROR_VERSION_NOT_SUPPORTED);
}
confess &log(
ERROR,
'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} .
' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" .
'HINT: is this version of PostgreSQL supported?',
ERROR_VERSION_NOT_SUPPORTED);
}
}

1
lib/pgBackRest/LibCAuto.pm
View File

@ -143,7 +143,6 @@ sub libcAutoExportTag
'CFGOPT_ARCHIVE_TIMEOUT',
'CFGOPT_BACKUP_STANDBY',
'CFGOPT_BUFFER_SIZE',
'CFGOPT_C',
'CFGOPT_CHECKSUM_PAGE',
'CFGOPT_CMD_SSH',
'CFGOPT_COMMAND',

196
lib/pgBackRest/Main.pm
View File

@ -1,196 +0,0 @@
####################################################################################################################################
# pgBackRest - Reliable PostgreSQL Backup & Restore
####################################################################################################################################
package pgBackRest::Main;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
# Convert die to confess to capture the stack trace
$SIG{__DIE__} = sub {Carp::confess @_};
use File::Basename qw(dirname);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
# Set config JSON separately to avoid exposing secrets in the stack trace
####################################################################################################################################
my $strConfigJson;
my $strConfigBin;
my $bConfigLoaded = false;
sub mainConfigSet
{
$strConfigBin = shift;
$strConfigJson = shift;
}
####################################################################################################################################
# Main entry point for the library
####################################################################################################################################
sub main
{
my $strCommand = shift;
my @stryCommandArg = @_;
# Run in eval block to catch errors
# ------------------------------------------------------------------------------------------------------------------------------
my $iResult = 0;
my $bErrorC = false;
my $strMessage = '';
eval
{
# Load command line parameters and config -- pass config by reference to hide secrets more than for efficiency
# --------------------------------------------------------------------------------------------------------------------------
if (!$bConfigLoaded)
{
configLoad(undef, $strConfigBin, $strCommand, \$strConfigJson);
$bConfigLoaded = true;
}
else
{
cfgCommandSet(cfgCommandId($strCommand));
}
# Process remote command
# --------------------------------------------------------------------------------------------------------------------------
if (cfgCommandTest(CFGCMD_REMOTE))
{
# Set log levels
cfgOptionSet(CFGOPT_LOG_LEVEL_STDERR, PROTOCOL, true);
logLevelSet(cfgOption(CFGOPT_LOG_LEVEL_FILE), OFF, cfgOption(CFGOPT_LOG_LEVEL_STDERR));
logFileSet(
storageLocal(),
cfgOption(CFGOPT_LOG_PATH) . '/' . (cfgOptionTest(CFGOPT_STANZA) ? cfgOption(CFGOPT_STANZA) : 'all') . '-' .
lc(cfgOption(CFGOPT_COMMAND)) . '-' . lc(cfgCommandName(cfgCommandGet())) . '-' .
sprintf("%03d", cfgOption(CFGOPT_PROCESS)));
if (cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP) &&
!cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3) &&
!-e cfgOption(CFGOPT_REPO_PATH))
{
confess &log(ERROR,
cfgOptionName(CFGOPT_REPO_PATH) . ' \'' . cfgOption(CFGOPT_REPO_PATH) . '\' does not exist',
ERROR_PATH_MISSING);
}
# Load module dynamically
require pgBackRest::Protocol::Remote::Minion;
pgBackRest::Protocol::Remote::Minion->import();
# Create the remote object
my $oRemote = new pgBackRest::Protocol::Remote::Minion(
cfgOption(CFGOPT_BUFFER_SIZE), cfgOption(CFGOPT_PROTOCOL_TIMEOUT));
# Process remote requests
$oRemote->process(
cfgOption(CFGOPT_LOCK_PATH), cfgOption(CFGOPT_COMMAND), cfgOption(CFGOPT_STANZA, false), cfgOption(CFGOPT_PROCESS));
}
else
{
# Check that the repo path exists
require pgBackRest::Protocol::Storage::Helper;
pgBackRest::Protocol::Storage::Helper->import();
if (isRepoLocal() && !cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3) && !storageRepo()->pathExists(''))
{
confess &log(ERROR,
cfgOptionName(CFGOPT_REPO_PATH) . ' \'' . cfgOption(CFGOPT_REPO_PATH) . '\' does not exist',
ERROR_PATH_MISSING);
}
logFileSet(
storageLocal(),
cfgOption(CFGOPT_LOG_PATH) . '/' . cfgOption(CFGOPT_STANZA) . '-' . lc(cfgCommandName(cfgCommandGet())));
}
return 1;
}
# Check for errors
# ------------------------------------------------------------------------------------------------------------------------------
or do
{
# Perl 5.10 seems to have a problem propagating errors up through a large call stack, so in the case that the error arrives
# blank just use the last logged error instead. Don't do this in all cases because newer Perls seem to work fine and there
# are other errors that could be arriving in $EVAL_ERROR.
my $oException = defined($EVAL_ERROR) && length($EVAL_ERROR) > 0 ? $EVAL_ERROR : logErrorLast();
# If a backrest exception
if (isException(\$oException))
{
$iResult = $oException->code();
$bErrorC = $oException->errorC();
# Only return message if we are in an async process since this will not be logged to the console
if (!$bConfigLoaded && cfgOption(CFGOPT_ARCHIVE_ASYNC))
{
$strMessage = $oException->message();
}
}
# Else a regular Perl exception
else
{
$iResult = ERROR_UNHANDLED;
$strMessage =
'process terminated due to an unhandled exception' .
(defined($oException) ? ":\n${oException}" : ': [exception not defined]');
}
};
# Return result and error message if the result is an error
return $iResult, $bErrorC, $strMessage;
}
####################################################################################################################################
# Do any cleanup required when the perl process is about to be shut down
####################################################################################################################################
sub mainCleanup
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$iExitCode,
) =
logDebugParam
(
__PACKAGE__ . '::mainCleanup', \@_,
{name => 'iExitCode', required => false},
);
# Don't fail if the remote can't be closed
eval
{
protocolDestroy(undef, undef, defined($iExitCode) && ($iExitCode == 0 || $iExitCode == 1));
return true;
}
# this eval exists only to suppress protocol shutdown errors so original error will not be lost
or do {};
# Don't fail if the lock can't be released (it will be freed by the system though the file will remain)
eval
{
lockRelease(false);
return true;
}
# this eval exists only to suppress lock errors so original error will not be lost
or do {};
# Log return values if any
return logDebugReturn($strOperation);
}
1;

5
lib/pgBackRest/Manifest.pm
View File

@ -19,7 +19,6 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
@ -1108,9 +1107,7 @@ sub build
# lead to an invalid diff/incr backup later when using timestamps to determine which files have changed. Offline backups do
# not wait because it makes testing much faster and Postgres should not be running (if it is the backup will not be
# consistent anyway and the one-second resolution problem is the least of our worries).
my $lTimeBegin =
$oStorageDbMaster->can('protocol') ?
$oStorageDbMaster->protocol()->cmdExecute(OP_WAIT, [$bOnline]) : waitRemainder($bOnline);
my $lTimeBegin = waitRemainder($bOnline);
# Check that links are valid
$self->linkCheck();

293
lib/pgBackRest/Protocol/Base/Master.pm
View File

@ -1,293 +0,0 @@
####################################################################################################################################
# Protocol Master Base
####################################################################################################################################
package pgBackRest::Protocol::Base::Master;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use Time::HiRes qw(gettimeofday);
use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Version;
####################################################################################################################################
# Operation constants
####################################################################################################################################
use constant OP_NOOP => 'noop';
push @EXPORT, qw(OP_NOOP);
use constant OP_EXIT => 'exit';
push @EXPORT, qw(OP_EXIT);
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strName},
$self->{strId},
$self->{oIo},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strName', trace => true},
{name => 'strId', trace => true},
{name => 'oIo', trace => true},
);
# Create JSON object
$self->{oJSON} = JSON::PP->new()->allow_nonref();
# Setup the keepalive timer
$self->{fKeepAliveTimeout} = $self->io()->timeout() / 2 > 120 ? 120 : $self->io()->timeout() / 2;
$self->{fKeepAliveTime} = gettimeofday();
# Set the error prefix used when raising error messages
$self->{strErrorPrefix} = 'raised from ' . $self->{strId};
# Check greeting to be sure the protocol matches
$self->greetingRead();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# DESTROY
####################################################################################################################################
sub DESTROY
{
my $self = shift;
$self->close();
}
####################################################################################################################################
# greetingRead
#
# Read the greeting and make sure it is as expected.
####################################################################################################################################
sub greetingRead
{
my $self = shift;
# Get the first line of output from the remote if possible
my $strGreeting = $self->io()->readLine(true);
# Parse the greeting and make sure it is valid
my $hGreeting;
eval
{
$hGreeting = $self->{oJSON}->decode($strGreeting);
return true;
}
# Report any error that stopped parsing
or do
{
$self->io()->error(ERROR_PROTOCOL, 'invalid protocol greeting', $strGreeting);
};
# Error if greeting parameters do not match
for my $hParam ({strName => 'name', strExpected => PROJECT_NAME},
{strName => 'version', strExpected => PROJECT_VERSION},
{strName => 'service', strExpected => $self->{strName}})
{
if (!defined($hGreeting->{$hParam->{strName}}) || $hGreeting->{$hParam->{strName}} ne $hParam->{strExpected})
{
confess &log(ERROR,
'found name \'' . (defined($hGreeting->{$hParam->{strName}}) ? $hGreeting->{$hParam->{strName}} : '[undef]') .
"' in protocol greeting instead of expected '$hParam->{strExpected}'", ERROR_HOST_CONNECT);
}
}
# Perform noop to catch errors early
$self->noOp();
}
####################################################################################################################################
# outputRead
#
# Read output from the remote process.
####################################################################################################################################
sub outputRead
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$bOutputRequired,
$bSuppressLog,
$bWarnOnError,
$bRef,
) =
logDebugParam
(
__PACKAGE__ . '->outputRead', \@_,
{name => 'bOutputRequired', default => false, trace => true},
{name => 'bSuppressLog', required => false, trace => true},
{name => 'bWarnOnError', default => false, trace => true},
{name => 'bRef', default => false, trace => true},
);
my $strProtocolResult = $self->io()->readLine();
logDebugMisc
(
$strOperation, undef,
{name => 'strProtocolResult', value => $strProtocolResult, trace => true}
);
my $hResult = $self->{oJSON}->decode($strProtocolResult);
# Raise any errors
if (defined($hResult->{err}))
{
my $strError = $self->{strErrorPrefix} . (defined($hResult->{out}) ? ": $hResult->{out}" : '');
# Raise the error if a warning is not requested
if (!$bWarnOnError)
{
confess &log(
ERROR, $strError . (defined($hResult->{errStack}) ? "\n$hResult->{errStack}" : ''), $hResult->{err}, $bSuppressLog);
}
&log(WARN, $strError, $hResult->{err});
undef($hResult->{out});
}
# Reset the keep alive time
$self->{fKeepAliveTime} = gettimeofday();
# If output is required and there is no output, raise exception
if ($bOutputRequired && !defined($hResult->{out}))
{
confess &log(ERROR, "$self->{strErrorPrefix}: output is not defined", ERROR_PROTOCOL_OUTPUT_REQUIRED);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hOutput', value => $hResult->{out}, ref => $bRef, trace => true}
);
}
####################################################################################################################################
# cmdWrite
#
# Send command to remote process.
####################################################################################################################################
sub cmdWrite
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCommand,
$hParam,
) =
logDebugParam
(
__PACKAGE__ . '->cmdWrite', \@_,
{name => 'strCommand', trace => true},
{name => 'hParam', required => false, trace => true},
);
my $strProtocolCommand = $self->{oJSON}->encode({cmd => $strCommand, param => $hParam});
logDebugMisc
(
$strOperation, undef,
{name => 'strProtocolCommand', value => $strProtocolCommand, trace => true}
);
# Write out the command
$self->io()->writeLine($strProtocolCommand);
# Reset the keep alive time
$self->{fKeepAliveTime} = gettimeofday();
# Return from function and log return values if any
logDebugReturn($strOperation);
}
####################################################################################################################################
# cmdExecute
#
# Send command to remote process and wait for output.
####################################################################################################################################
sub cmdExecute
{
my $self = shift;
my $strCommand = shift;
my $oParamRef = shift;
my $bOutputRequired = shift;
my $bWarnOnError = shift;
$self->cmdWrite($strCommand, $oParamRef);
return $self->outputRead($bOutputRequired, undef, $bWarnOnError);
}
####################################################################################################################################
# keepAlive
#
# Send periodic noops so the remote does not time out.
####################################################################################################################################
sub keepAlive
{
my $self = shift;
if (gettimeofday() - $self->{fKeepAliveTimeout} > $self->{fKeepAliveTime})
{
$self->noOp();
}
}
####################################################################################################################################
# noOp
#
# Send noop to test connection or keep it alive.
####################################################################################################################################
sub noOp
{
my $self = shift;
$self->cmdExecute(OP_NOOP, undef, false);
$self->{fKeepAliveTime} = gettimeofday();
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub io {shift->{oIo}}
sub master {true}
1;

247
lib/pgBackRest/Protocol/Base/Minion.pm
View File

@ -1,247 +0,0 @@
####################################################################################################################################
# Protocol Minion Base
####################################################################################################################################
package pgBackRest::Protocol::Base::Minion;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::LibC qw(:lock);
use pgBackRest::Protocol::Base::Master;
use pgBackRest::Protocol::Helper;
use pgBackRest::Version;
####################################################################################################################################
# Constant used to define code to run after each operation
####################################################################################################################################
use constant OP_POST => 'post';
push @EXPORT, qw(OP_POST);
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strName},
$self->{oIo},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strName', trace => true},
{name => 'oIo', trace => true},
);
# Create JSON object
$self->{oJSON} = JSON::PP->new()->allow_nonref();
# Write the greeting so master process knows who we are
$self->greetingWrite();
# Initialize module variables
$self->{hCommandMap} = $self->can('init') ? $self->init() : undef;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# greetingWrite
#
# Send a greeting to the master process.
####################################################################################################################################
sub greetingWrite
{
my $self = shift;
# Write the greeting
$self->io()->writeLine((JSON::PP->new()->canonical()->allow_nonref())->encode(
{name => PROJECT_NAME, service => $self->{strName}, version => PROJECT_VERSION}));
}
####################################################################################################################################
# errorWrite
#
# Write errors with error codes in protocol format, otherwise write to stderr and exit with error.
####################################################################################################################################
sub errorWrite
{
my $self = shift;
my $oException = shift;
# Throw hard error if this is not a standard exception
if (!isException(\$oException))
{
confess &log(ERROR, 'unknown error: ' . $oException, ERROR_UNKNOWN);
}
# Write error code and message
$self->io()->writeLine($self->{oJSON}->encode({err => $oException->code(), out => $oException->message()}));
}
####################################################################################################################################
# outputWrite
#
# Write output for the master process.
####################################################################################################################################
sub outputWrite
{
my $self = shift;
$self->io()->writeLine($self->{oJSON}->encode({out => \@_}));
}
####################################################################################################################################
# cmdRead
#
# Read command sent by the master process.
####################################################################################################################################
sub cmdRead
{
my $self = shift;
my $hCommand = $self->{oJSON}->decode($self->io()->readLine());
return $hCommand->{cmd}, $hCommand->{param};
}
####################################################################################################################################
# process
####################################################################################################################################
sub process
{
my $self = shift;
my $strLockPath = shift;
my $strLockCommand = shift;
my $strLockStanza = shift;
my $iProcessId = shift;
# Reset stderr log level so random errors do not get output
logLevelSet(undef, undef, OFF);
# A permanent error will be returned from every command once it has been set. In some cases this is a more graceful way to
# exit than a hard error.
my $oPermanentError;
# Loop until the exit command is received
eval
{
# Acquire a lock if required (this will be determined by lockAcquire()). This is done here so any errors will be
# transmitted through the protocol layer and cause a graceful shutdown rather than a remote abort.
if (defined($strLockPath) && defined($strLockStanza) && $iProcessId == 0)
{
eval
{
if (lockAcquire($strLockPath, $strLockCommand, $strLockStanza, 30, true))
{
# Check if processes have been stopped
lockStopTest();
}
return true;
}
or do
{
$oPermanentError = $EVAL_ERROR;
};
}
while (true)
{
my ($strCommand, $rParam) = $self->cmdRead();
last if ($strCommand eq OP_EXIT);
# If permanent error is set then always return it
if (defined($oPermanentError))
{
$self->errorWrite($oPermanentError);
}
# Else process as usual
else
{
eval
{
# Check for the command in the map and run it if found
if (defined($self->{hCommandMap}{$strCommand}))
{
$self->outputWrite($self->{hCommandMap}{$strCommand}->($rParam));
}
# Run the standard NOOP command. This this can be overridden in hCommandMap to implement a custom NOOP.
elsif ($strCommand eq OP_NOOP)
{
protocolKeepAlive();
$self->outputWrite();
}
else
{
confess "invalid command: ${strCommand}";
}
# Run the post command if defined
if (defined($self->{hCommandMap}{&OP_POST}))
{
$self->{hCommandMap}{&OP_POST}->();
}
return true;
}
# Process errors
or do
{
$self->errorWrite($EVAL_ERROR);
};
}
}
return true;
}
or do
{
my $oException = $EVAL_ERROR;
# Change log level so error will go to stderr
logLevelSet(undef, undef, PROTOCOL);
# If standard exception
if (isException(\$oException))
{
confess &log($oException->level(), $oException->message(), $oException->code());
}
# Else unexpected Perl exception
confess &log(ERROR, 'unknown error: ' . $oException, ERROR_UNKNOWN);
};
}
####################################################################################################################################
# Getters
####################################################################################################################################
sub io {shift->{oIo}}
sub master {false}
1;

148
lib/pgBackRest/Protocol/Command/Master.pm
View File

@ -1,148 +0,0 @@
####################################################################################################################################
# PROTOCOL COMMAND MASTER MODULE
####################################################################################################################################
package pgBackRest::Protocol::Command::Master;
use parent 'pgBackRest::Protocol::Base::Master';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(dirname);
use Time::HiRes qw(gettimeofday);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Io::Process;
use pgBackRest::Protocol::Base::Master;
use pgBackRest::Version;
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strName, # Name of the protocol
$strId, # Id of this process for error messages
$strCommand, # Command to execute on local/remote
$iBufferMax, # Maximum buffer size
$iCompressLevel, # Set compression level
$iCompressLevelNetwork, # Set compression level for network only compression
$iProtocolTimeout, # Protocol timeout
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strName'},
{name => 'strId'},
{name => 'strCommand'},
{name => 'iBufferMax'},
{name => 'iCompressLevel'},
{name => 'iCompressLevelNetwork'},
{name => 'iProtocolTimeout'},
);
# Set command
if (!defined($strCommand))
{
confess &log(ASSERT, 'strCommand must be set');
}
# Execute the command
my $oIo = new pgBackRest::Common::Io::Process(
new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle($strId), $iProtocolTimeout, $iBufferMax), $strCommand);
# Create the class hash
my $self = $class->SUPER::new($strName, $strId, $oIo);
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# close
####################################################################################################################################
sub close
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$bComplete,
) =
logDebugParam
(
__PACKAGE__ . '->close', \@_,
{name => 'bComplete', default => false, trace => true},
);
# Exit status defaults to success
my $iExitStatus = 0;
my $bClosed = false;
# Only send the exit command if the process is running
if (defined($self->io()) && defined($self->io()->processId()))
{
&log(TRACE, "sending exit command to process");
eval
{
$self->cmdWrite(OP_EXIT);
return true;
}
or do
{
my $oException = $EVAL_ERROR;
my $strError = 'unable to shutdown protocol';
my $strHint = 'HINT: the process completed all operations successfully but protocol-timeout may need to be increased.';
if (isException(\$oException))
{
$iExitStatus = $oException->code();
}
else
{
if (!defined($oException))
{
$oException = 'unknown error';
}
$iExitStatus = ERROR_UNKNOWN;
}
&log(WARN,
$strError . ($iExitStatus == ERROR_UNKNOWN ? '' : sprintf(' [%03d]', $oException->code())) . ': ' .
($iExitStatus == ERROR_UNKNOWN ? $oException : $oException->message()) .
($bComplete ? "\n${strHint}" : ''));
};
$self->{oIo}->close();
undef($self->{oIo});
$bClosed = true;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iExitStatus', value => $iExitStatus, trace => !$bClosed}
);
}
1;

62
lib/pgBackRest/Protocol/Command/Minion.pm
View File

@ -1,62 +0,0 @@
####################################################################################################################################
# PROTOCOL COMMAND MINION MODULE
####################################################################################################################################
package pgBackRest::Protocol::Command::Minion;
use parent 'pgBackRest::Protocol::Base::Minion';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Protocol::Base::Minion;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Version;
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strName, # Name of the protocol
$iBufferMax, # Maximum buffer size
$iProtocolTimeout, # Protocol timeout
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strName'},
{name => 'iBufferMax'},
{name => 'iProtocolTimeout'},
);
# Open buffered protocol io
my $oIo =
new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('stdio', *STDIN, *STDOUT), $iProtocolTimeout, $iBufferMax);
# Create the class hash
my $self = $class->SUPER::new($strName, $oIo);
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
1;

489
lib/pgBackRest/Protocol/Helper.pm
View File

@ -1,489 +0,0 @@
####################################################################################################################################
# Create and manage protocol objects.
####################################################################################################################################
package pgBackRest::Protocol::Helper;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Remote::Master;
use pgBackRest::Version;
####################################################################################################################################
# Operation constants
####################################################################################################################################
# Backup module
use constant OP_BACKUP_FILE => 'backupFile';
push @EXPORT, qw(OP_BACKUP_FILE);
# Archive Module
use constant OP_ARCHIVE_GET_CHECK => 'archiveCheck';
push @EXPORT, qw(OP_ARCHIVE_GET_CHECK);
# Db Module
use constant OP_DB_CONNECT => 'dbConnect';
push @EXPORT, qw(OP_DB_CONNECT);
use constant OP_DB_EXECUTE_SQL => 'dbExecSql';
push @EXPORT, qw(OP_DB_EXECUTE_SQL);
use constant OP_DB_INFO => 'dbInfo';
push @EXPORT, qw(OP_DB_INFO);
# Storage Module
use constant OP_STORAGE_OPEN_READ => 'storageOpenRead';
push @EXPORT, qw(OP_STORAGE_OPEN_READ);
use constant OP_STORAGE_OPEN_WRITE => 'storageOpenWrite';
push @EXPORT, qw(OP_STORAGE_OPEN_WRITE);
use constant OP_STORAGE_CIPHER_PASS_USER => 'storageCipherPassUser';
push @EXPORT, qw(OP_STORAGE_CIPHER_PASS_USER);
use constant OP_STORAGE_EXISTS => 'storageExists';
push @EXPORT, qw(OP_STORAGE_EXISTS);
use constant OP_STORAGE_HASH_SIZE => 'storageHashSize';
push @EXPORT, qw(OP_STORAGE_HASH_SIZE);
use constant OP_STORAGE_LIST => 'storageList';
push @EXPORT, qw(OP_STORAGE_LIST);
use constant OP_STORAGE_MANIFEST => 'storageManifest';
push @EXPORT, qw(OP_STORAGE_MANIFEST);
use constant OP_STORAGE_MOVE => 'storageMove';
push @EXPORT, qw(OP_STORAGE_MOVE);
use constant OP_STORAGE_PATH_GET => 'storagePathGet';
push @EXPORT, qw(OP_STORAGE_PATH_GET);
# Restore module
use constant OP_RESTORE_FILE => 'restoreFile';
push @EXPORT, qw(OP_RESTORE_FILE);
# Wait
use constant OP_WAIT => 'wait';
push @EXPORT, qw(OP_WAIT);
####################################################################################################################################
# Module variables
####################################################################################################################################
my $hProtocol = {}; # Global remote hash that is created on first request
####################################################################################################################################
# isRepoLocal
#
# Is the backup/archive repository local?
####################################################################################################################################
sub isRepoLocal
{
# Not valid for remote
if (cfgCommandTest(CFGCMD_REMOTE) && !cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP))
{
confess &log(ASSERT, 'isRepoLocal() not valid on ' . cfgOption(CFGOPT_TYPE) . ' remote');
}
return cfgOptionTest(CFGOPT_REPO_HOST) ? false : true;
}
push @EXPORT, qw(isRepoLocal);
####################################################################################################################################
# isDbLocal - is the database local?
####################################################################################################################################
sub isDbLocal
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$iRemoteIdx,
) =
logDebugParam
(
__PACKAGE__ . '::isDbLocal', \@_,
{name => 'iRemoteIdx', optional => true, default => cfgOptionValid(CFGOPT_HOST_ID) ? cfgOption(CFGOPT_HOST_ID) : 1,
trace => true},
);
# Not valid for remote
if (cfgCommandTest(CFGCMD_REMOTE) && !cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_DB))
{
confess &log(ASSERT, 'isDbLocal() not valid on ' . cfgOption(CFGOPT_TYPE) . ' remote');
}
my $bLocal = cfgOptionTest(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iRemoteIdx)) ? false : true;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bLocal', value => $bLocal, trace => true}
);
}
push @EXPORT, qw(isDbLocal);
####################################################################################################################################
# Gets the parameters required to setup the protocol
####################################################################################################################################
sub protocolParam
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCommand,
$strRemoteType,
$iRemoteIdx,
$strBackRestBin,
$iProcessIdx,
) =
logDebugParam
(
__PACKAGE__ . '::protocolParam', \@_,
{name => 'strCommand'},
{name => 'strRemoteType'},
{name => 'iRemoteIdx', default => cfgOptionValid(CFGOPT_HOST_ID) ? cfgOption(CFGOPT_HOST_ID) : 1},
{name => 'strBackRestBin', optional => true},
{name => 'iProcessIdx', optional => true,
default => cfgOptionValid(CFGOPT_PROCESS) ? cfgOption(CFGOPT_PROCESS, false) : undef},
);
# Return the remote when required
my $iOptionIdCmd = CFGOPT_REPO_HOST_CMD;
my $iOptionIdConfig = CFGOPT_REPO_HOST_CONFIG;
my $iOptionIdConfigIncludePath = CFGOPT_REPO_HOST_CONFIG_INCLUDE_PATH;
my $iOptionIdConfigPath = CFGOPT_REPO_HOST_CONFIG_PATH;
my $iOptionIdHost = CFGOPT_REPO_HOST;
my $iOptionIdUser = CFGOPT_REPO_HOST_USER;
my $strOptionDbPath = undef;
my $strOptionDbPort = undef;
my $strOptionDbSocketPath = undef;
my $strOptionSshPort = CFGOPT_REPO_HOST_PORT;
if ($strRemoteType eq CFGOPTVAL_REMOTE_TYPE_DB)
{
$iOptionIdCmd = cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iRemoteIdx);
$iOptionIdConfig = cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iRemoteIdx);
$iOptionIdConfigIncludePath = cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH, $iRemoteIdx);
$iOptionIdConfigPath = cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_PATH, $iRemoteIdx);
$iOptionIdHost = cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iRemoteIdx);
$iOptionIdUser = cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iRemoteIdx);
$strOptionSshPort = cfgOptionIdFromIndex(CFGOPT_PG_HOST_PORT, $iRemoteIdx);
}
# Db path is not valid in all contexts (restore, for instance)
if (cfgOptionValid(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx)))
{
$strOptionDbPath =
cfgOptionSource(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx)) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iRemoteIdx));
}
# Db port is not valid in all contexts (restore, for instance)
if (cfgOptionValid(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iRemoteIdx)))
{
$strOptionDbPort =
cfgOptionSource(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iRemoteIdx)) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iRemoteIdx));
}
# Db socket is not valid in all contexts (restore, for instance)
if (cfgOptionValid(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $iRemoteIdx)))
{
$strOptionDbSocketPath =
cfgOptionSource(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $iRemoteIdx)) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption(cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $iRemoteIdx));
}
# Build hash to set and override command options
my $rhCommandOption =
{
&CFGOPT_COMMAND => {value => $strCommand},
&CFGOPT_PROCESS => {value => defined($iProcessIdx) ? $iProcessIdx : 0},
&CFGOPT_CONFIG =>
{value => cfgOptionValid($iOptionIdConfig) && cfgOptionSource($iOptionIdConfig) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption($iOptionIdConfig)},
&CFGOPT_CONFIG_INCLUDE_PATH =>
{value => cfgOptionValid($iOptionIdConfigIncludePath) &&
cfgOptionSource($iOptionIdConfigIncludePath) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption($iOptionIdConfigIncludePath)},
&CFGOPT_CONFIG_PATH =>
{value => cfgOptionValid($iOptionIdConfigPath) && cfgOptionSource($iOptionIdConfigPath) eq CFGDEF_SOURCE_DEFAULT ?
undef : cfgOption($iOptionIdConfigPath)},
&CFGOPT_TYPE => {value => $strRemoteType},
&CFGOPT_LOG_PATH => {},
&CFGOPT_LOCK_PATH => {},
# Only enable file logging on the remote when requested
&CFGOPT_LOG_LEVEL_FILE => {value => cfgOption(CFGOPT_LOG_SUBPROCESS) ? cfgOption(CFGOPT_LOG_LEVEL_FILE) : lc(OFF)},
# Don't pass CFGOPT_LOG_LEVEL_STDERR because in the case of the local process calling the remote process the
# option will be set to 'protocol' which is not a valid value from the command line.
&CFGOPT_LOG_LEVEL_STDERR => {},
cfgOptionIdFromIndex(CFGOPT_PG_PATH, 1) => {value => $strOptionDbPath},
cfgOptionIdFromIndex(CFGOPT_PG_PORT, 1) => {value => $strOptionDbPort},
cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, 1) => {value => $strOptionDbSocketPath},
# Set protocol options explicitly so values are not picked up from remote config files
&CFGOPT_BUFFER_SIZE => {value => cfgOption(CFGOPT_BUFFER_SIZE)},
&CFGOPT_COMPRESS_LEVEL => {value => cfgOption(CFGOPT_COMPRESS_LEVEL)},
&CFGOPT_COMPRESS_LEVEL_NETWORK => {value => cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK)},
&CFGOPT_PROTOCOL_TIMEOUT => {value => cfgOption(CFGOPT_PROTOCOL_TIMEOUT)}
};
# Override some per-db options that shouldn't be passed to the command. ??? This could be done better as a new define
# for these options which would then be implemented in cfgCommandWrite().
for (my $iOptionIdx = 1; $iOptionIdx <= cfgOptionIndexTotal(CFGOPT_PG_HOST); $iOptionIdx++)
{
if ($iOptionIdx != 1)
{
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_PATH, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_SOCKET_PATH, $iOptionIdx)} = {};
}
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iOptionIdx)} = {};
$rhCommandOption->{cfgOptionIdFromIndex(CFGOPT_PG_HOST_PORT, $iOptionIdx)} = {};
}
# Generate the remote command
my $strRemoteCommand = cfgCommandWrite(
CFGCMD_REMOTE, true, defined($strBackRestBin) ? $strBackRestBin : cfgOption($iOptionIdCmd), undef, $rhCommandOption);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strRemoteHost', value => cfgOption($iOptionIdHost)},
{name => 'strRemoteHostUser', value => cfgOption($iOptionIdUser)},
{name => 'strRemoteHostSshPort', value => cfgOption($strOptionSshPort, false)},
{name => 'strRemoteCommand', value => $strRemoteCommand},
);
}
####################################################################################################################################
# protocolGet
#
# Get the protocol object or create it if does not exist. Shared protocol objects are used because they create an SSH connection
# to the remote host and the number of these connections should be minimized.
####################################################################################################################################
sub protocolGet
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strRemoteType,
$iRemoteIdx,
$bCache,
$strBackRestBin,
$iProcessIdx,
$strCommand,
) =
logDebugParam
(
__PACKAGE__ . '::protocolGet', \@_,
{name => 'strRemoteType'},
{name => 'iRemoteIdx', default => cfgOptionValid(CFGOPT_HOST_ID) ? cfgOption(CFGOPT_HOST_ID) : 1},
{name => 'bCache', optional => true, default => true},
{name => 'strBackRestBin', optional => true},
{name => 'iProcessIdx', optional => true},
{name => 'strCommand', optional => true,
default => cfgOptionValid(CFGOPT_COMMAND) ? cfgOption(CFGOPT_COMMAND) : cfgCommandName(cfgCommandGet())},
);
# Protocol object
my $oProtocol;
# If no remote requested or if the requested remote type is local then return a local protocol object
if (!cfgOptionTest(
cfgOptionIdFromIndex($strRemoteType eq CFGOPTVAL_REMOTE_TYPE_DB ? CFGOPT_PG_HOST : CFGOPT_REPO_HOST, $iRemoteIdx)))
{
confess &log(ASSERT, 'protocol cannot be created when remote host is not specified');
}
# Else create the remote protocol
else
{
# Set protocol to cached value
$oProtocol =
$bCache && defined($$hProtocol{$strRemoteType}{$iRemoteIdx}) ? $$hProtocol{$strRemoteType}{$iRemoteIdx} : undef;
if ($bCache && $$hProtocol{$strRemoteType}{$iRemoteIdx})
{
$oProtocol = $$hProtocol{$strRemoteType}{$iRemoteIdx};
logDebugMisc($strOperation, 'found cached protocol');
# Issue a noop on protocol pulled from the cache to be sure it is still functioning. It's better to get an error at
# request time than somewhere randomly later.
$oProtocol->noOp();
}
# If protocol was not returned from cache then create it
if (!defined($oProtocol))
{
logDebugMisc($strOperation, 'create (' . ($bCache ? '' : 'un') . 'cached) remote protocol');
my ($strRemoteHost, $strRemoteHostUser, $strRemoteHostSshPort, $strRemoteCommand) = protocolParam(
$strCommand, $strRemoteType, $iRemoteIdx, {strBackRestBin => $strBackRestBin, iProcessIdx => $iProcessIdx});
$oProtocol = new pgBackRest::Protocol::Remote::Master
(
cfgOption(CFGOPT_CMD_SSH),
$strRemoteCommand,
cfgOption(CFGOPT_BUFFER_SIZE),
cfgOption(CFGOPT_COMPRESS_LEVEL),
cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK),
$strRemoteHost,
$strRemoteHostUser,
$strRemoteHostSshPort,
cfgOption(CFGOPT_PROTOCOL_TIMEOUT)
);
# Cache the protocol
if ($bCache)
{
$$hProtocol{$strRemoteType}{$iRemoteIdx} = $oProtocol;
}
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oProtocol', value => $oProtocol, trace => true}
);
}
push @EXPORT, qw(protocolGet);
####################################################################################################################################
# protocolList - list all active protocols
####################################################################################################################################
sub protocolList
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strRemoteType,
$iRemoteIdx,
) =
logDebugParam
(
__PACKAGE__ . '::protocolList', \@_,
{name => 'strRemoteType', required => false, trace => true},
{name => 'iRemoteIdx', required => false, trace => true},
);
my @oyProtocol;
if (!defined($strRemoteType))
{
foreach my $strRemoteType (sort(keys(%{$hProtocol})))
{
push(@oyProtocol, protocolList($strRemoteType));
}
}
elsif (!defined($iRemoteIdx))
{
foreach my $iRemoteIdx (sort(keys(%{$hProtocol->{$strRemoteType}})))
{
push(@oyProtocol, protocolList($strRemoteType, $iRemoteIdx));
}
}
elsif (defined($hProtocol->{$strRemoteType}{$iRemoteIdx}))
{
push(@oyProtocol, {strRemoteType => $strRemoteType, iRemoteIdx => $iRemoteIdx});
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oyProtocol', value => \@oyProtocol, trace => true}
);
}
####################################################################################################################################
# protocolDestroy
#
# Undefine the protocol if it is stored locally.
####################################################################################################################################
sub protocolDestroy
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strRemoteType,
$iRemoteIdx,
$bComplete,
) =
logDebugParam
(
__PACKAGE__ . '::protocolDestroy', \@_,
{name => 'strRemoteType', required => false},
{name => 'iRemoteIdx', required => false},
{name => 'bComplete', default => false},
);
my $iExitStatus = 0;
foreach my $rhProtocol (protocolList($strRemoteType, $iRemoteIdx))
{
logDebugMisc(
$strOperation, 'found cached protocol',
{name => 'strRemoteType', value => $rhProtocol->{strRemoteType}},
{name => 'iRemoteIdx', value => $rhProtocol->{iRemoteIdx}});
$iExitStatus = $hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}}->close($bComplete);
delete($hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}});
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iExitStatus', value => $iExitStatus}
);
}
push @EXPORT, qw(protocolDestroy);
####################################################################################################################################
# protocolKeepAlive - call keepAlive() on all protocols
####################################################################################################################################
sub protocolKeepAlive
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strRemoteType,
$iRemoteIdx,
) =
logDebugParam
(
__PACKAGE__ . '::protocolDestroy', \@_,
{name => 'strRemoteType', required => false, trace => true},
{name => 'iRemoteIdx', required => false, trace => true},
);
foreach my $rhProtocol (protocolList($strRemoteType, $iRemoteIdx))
{
$hProtocol->{$rhProtocol->{strRemoteType}}{$rhProtocol->{iRemoteIdx}}->keepAlive();
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push @EXPORT, qw(protocolKeepAlive);
1;

51
lib/pgBackRest/Protocol/Local/Master.pm
View File

@ -1,51 +0,0 @@
####################################################################################################################################
# PROTOCOL LOCAL MASTER MODULE
####################################################################################################################################
package pgBackRest::Protocol::Local::Master;
use parent 'pgBackRest::Protocol::Command::Master';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Command::Master;
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCommand,
$iProcessIdx,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strCommand'},
{name => 'iProcessIdx', default => 1},
);
# Init object and store variables
my $self = $class->SUPER::new(
'local', "local-${iProcessIdx} process", $strCommand, cfgOption(CFGOPT_BUFFER_SIZE),
cfgOption(CFGOPT_COMPRESS_LEVEL), cfgOption(CFGOPT_COMPRESS_LEVEL_NETWORK), cfgOption(CFGOPT_PROTOCOL_TIMEOUT));
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
1;

667
lib/pgBackRest/Protocol/Local/Process.pm
View File

@ -1,667 +0,0 @@
####################################################################################################################################
# PROTOCOL LOCAL PROCESS MODULE
#
# This module can be extended by commands that want to perform jobs in parallel.
####################################################################################################################################
package pgBackRest::Protocol::Local::Process;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use IO::Select;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Local::Master;
use pgBackRest::Version;
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift;
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strHostType},
$self->{iSelectTimeout},
$self->{strBackRestBin},
$self->{bConfessError},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strHostType'},
{name => 'iSelectTimeout', default => int(cfgOption(CFGOPT_PROTOCOL_TIMEOUT) / 2)},
{name => 'strBackRestBin', default => projectBin()},
{name => 'bConfessError', default => true},
);
# Declare host map and array
$self->{hHostMap} = {};
$self->{hyHost} = undef;
# Reset module variables to get ready for queueing
$self->reset();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# reset
#
# Reset to initial state.
####################################################################################################################################
sub reset
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->reset');
# Select object is used to check for new results
$self->{oSelect} = undef;
# Declare local process map and array
$self->{hyLocalMap} = undef;
$self->{hyLocal} = undef;
# Set the processing flag to false
$self->{bProcessing} = false;
# Initialize job total to 0
$self->{iQueued} = 0;
# Initialize running job total to 0
$self->{iRunning} = 0;
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# hostAdd
#
# Add a host where jobs can be executed.
####################################################################################################################################
sub hostAdd
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$iHostConfigIdx,
$iProcessMax,
) =
logDebugParam
(
__PACKAGE__ . '->hostAdd', \@_,
{name => 'iHostConfigIdx'},
{name => 'iProcessMax'},
);
my $iHostIdx = $self->{hHostMap}{$iHostConfigIdx};
if (!defined($iHostIdx))
{
$iHostIdx = defined($self->{hyHost}) ? @{$self->{hyHost}} : 0;
$self->{hHostMap}{$iHostConfigIdx} = $iHostIdx;
}
my $hHost =
{
iHostConfigIdx => $iHostConfigIdx,
iProcessMax => $iProcessMax,
};
push(@{$self->{hyHost}}, $hHost);
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# hostConnect
#
# Connect local processes to the hosts.
####################################################################################################################################
sub hostConnect
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->hostConnect');
# Create a select object used to monitor output from minions
$self->{oSelect} = IO::Select->new();
# Iterate hosts
my $iHostIdx = 0;
foreach my $hHost (@{$self->{hyHost}})
{
# If there are no jobs in the queue for this host then no need to connect
if (!defined($hHost->{hyQueue}))
{
logDebugMisc(
$strOperation, "no jobs for host",
{name => 'strHostType', value => $self->{strHostType}},
{name => 'iHostConfigIdx', value => $hHost->{iHostConfigIdx}});
next;
}
for (my $iHostProcessIdx = 0; $iHostProcessIdx < $hHost->{iProcessMax}; $iHostProcessIdx++)
{
my $iLocalIdx = defined($self->{hyLocal}) ? @{$self->{hyLocal}} : 0;
my $iProcessId = $iLocalIdx + 1;
logDebugMisc(
$strOperation, 'start local process',
{name => 'strHostType', value => $self->{strHostType}},
{name => 'iHostProcessIdx', value => $iHostProcessIdx},
{name => 'iHostConfigIdx', value => $hHost->{iHostConfigIdx}},
{name => 'iHostIdx', value => $iHostIdx},
{name => 'iProcessId', value => $iProcessId});
my $oLocal = new pgBackRest::Protocol::Local::Master
(
cfgCommandWrite(
CFGCMD_LOCAL, true, $self->{strBackRestBin}, undef,
{
&CFGOPT_COMMAND => {value => cfgCommandName(cfgCommandGet())},
&CFGOPT_PROCESS => {value => $iProcessId},
&CFGOPT_TYPE => {value => $self->{strHostType}},
&CFGOPT_HOST_ID => {value => $hHost->{iHostConfigIdx}},
# Only enable file logging on the local when requested
&CFGOPT_LOG_LEVEL_FILE =>
{value => cfgOption(CFGOPT_LOG_SUBPROCESS) ? cfgOption(CFGOPT_LOG_LEVEL_FILE) : lc(OFF)},
&CFGOPT_LOG_LEVEL_STDERR => {},
}),
$iLocalIdx + 1
);
my $hLocal =
{
iHostIdx => $iHostIdx,
iProcessId => $iProcessId,
iHostProcessIdx => $iHostProcessIdx,
oLocal => $oLocal,
hndIn => fileno($oLocal->io()->handleRead()),
};
push(@{$self->{hyLocal}}, $hLocal);
$self->{hLocalMap}{$hLocal->{hndIn}} = $hLocal;
$self->{oSelect}->add($hLocal->{hndIn});
}
$iHostIdx++;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => $iHostIdx > 0 ? true : false}
);
}
####################################################################################################################################
# init
#
# Initialize data structures required for processing.
####################################################################################################################################
sub init
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->init');
if ($self->hostConnect())
{
foreach my $hLocal (@{$self->{hyLocal}})
{
my $hHost = $self->{hyHost}[$hLocal->{iHostIdx}];
my $hyQueue = $hHost->{hyQueue};
# Initialize variables to keep track of what job the local is working on
$hLocal->{iDirection} = $hLocal->{iHostProcessIdx} % 2 == 0 ? 1 : -1;
$hLocal->{iQueueIdx} = int((@{$hyQueue} / $hHost->{iProcessMax}) * $hLocal->{iHostProcessIdx});
# Calculate the last queue that this process should pull from
$hLocal->{iQueueLastIdx} = $hLocal->{iQueueIdx} + ($hLocal->{iDirection} * -1);
if ($hLocal->{iQueueLastIdx} < 0)
{
$hLocal->{iQueueLastIdx} = @{$hyQueue} - 1;
}
elsif ($hLocal->{iQueueLastIdx} >= @{$hyQueue})
{
$hLocal->{iQueueLastIdx} = 0;
}
logDebugMisc(
$strOperation, 'init local process',
{name => 'iHostIdx', value => $hLocal->{iHostIdx}},
{name => 'iProcessId', value => $hLocal->{iProcessId}},
{name => 'iDirection', value => $hLocal->{iDirection}},
{name => 'iQueueIdx', value => $hLocal->{iQueueIdx}},
{name => 'iQueueLastIdx', value => $hLocal->{iQueueLastIdx}});
}
$self->{bProcessing} = true;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => $self->processing()}
);
}
####################################################################################################################################
# process
#
# Run all jobs and return results in batches.
####################################################################################################################################
sub process
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->process');
# Initialize processing
if (!$self->processing())
{
if (!$self->init())
{
logDebugMisc($strOperation, 'no jobs to run');
$self->reset();
return;
}
}
# If jobs are running then wait for any of them to complete
my @hyResult = ();
my $iCompleted = 0;
if ($self->{iRunning} > 0)
{
&logDebugMisc(
$strOperation, 'check running jobs',
{name => 'iRunning', value => $self->{iRunning}, trace => true});
# Wait for results to be available on any of the local process inputs
my @hndyIn = $self->{oSelect}->can_read($self->{iSelectTimeout});
# Fetch results from the completed jobs
foreach my $hndIn (@hndyIn)
{
# Get the local data
my $hLocal = $self->{hLocalMap}{$hndIn};
if (!defined($hLocal))
{
confess &log(ASSERT, "unable to map from fileno ${hndIn} to local");
}
# Get the job result
my $hJob = $hLocal->{hJob};
eval
{
$hJob->{rResult} = $hLocal->{oLocal}->outputRead(true, undef, undef, true);
# Create a result array when the result is not already an array. The Perl locals always return an array but the C
# locals only do so when needed.
if (ref($hJob->{rResult}) ne 'ARRAY')
{
my @resultArray = (${$hJob->{rResult}});
$hJob->{rResult} = \@resultArray;
}
return true;
}
or do
{
my $oException = $EVAL_ERROR;
# If not a backrest exception then always confess it - something has gone very wrong
confess $oException if (!isException(\$oException));
# If the process is has terminated throw the exception
if (!defined($hLocal->{oLocal}->io()->processId()))
{
confess logException($oException);
}
# If errors should be confessed then do so
if ($self->{bConfessError})
{
confess logException($oException);
}
# Else store exception so caller can process it
else
{
$hJob->{oException} = $oException;
}
};
$hJob->{iProcessId} = $hLocal->{iProcessId};
push(@hyResult, $hJob);
logDebugMisc(
$strOperation, 'job complete',
{name => 'iProcessId', value => $hJob->{iProcessId}},
{name => 'strKey', value => $hJob->{strKey}},
{name => 'rResult', value => $hJob->{rResult}});
# Free the local process to receive another job
$hLocal->{hJob} = undef;
$self->{iRunning}--;
$iCompleted++;
}
}
# If any jobs are not running/completed then assign new jobs
if ($self->{iRunning} == 0 || $iCompleted > 0)
{
&logDebugMisc(
$strOperation, 'get new jobs',
{name => 'iRunning', value => $self->{iRunning}, trace => true},
{name => 'iCompleted', value => $iCompleted, trace => true});
my $bFound = false;
my $iLocalIdx = -1;
# Iterate all local processes
foreach my $hLocal (@{$self->{hyLocal}})
{
# Skip this local process if it has already completed
$iLocalIdx++;
next if (!defined($hLocal));
my $hHost = $self->{hyHost}[$hLocal->{iHostIdx}];
my $hyQueue = $hHost->{hyQueue};
# If this process does not currently have a job assigned then find one
if (!defined($hLocal->{hJob}))
{
# Search queues for a new job
my $iQueueIdx = $hLocal->{iQueueIdx};
my $hJob = shift(@{$$hyQueue[$iQueueIdx]});
while (!defined($hJob) && $iQueueIdx != $hLocal->{iQueueLastIdx})
{
$iQueueIdx += $hLocal->{iDirection};
if ($iQueueIdx < 0)
{
$iQueueIdx = @{$hyQueue} - 1;
}
elsif ($iQueueIdx >= @{$hyQueue})
{
$iQueueIdx = 0;
}
$hJob = shift(@{$$hyQueue[$iQueueIdx]});
}
# If no job was found then stop the local process
if (!defined($hJob))
{
logDebugMisc(
$strOperation, 'no jobs found, stop local',
{name => 'strHostType', value => $hLocal->{strHostType}},
{name => 'iHostConfigIdx', value => $hLocal->{iHostConfigIdx}},
{name => 'iHostIdx', value => $hLocal->{iHostIdx}},
{name => 'iProcessId', value => $hLocal->{iProcessId}});
# Remove input handle from the select object
my $iHandleTotal = $self->{oSelect}->count();
$self->{oSelect}->remove($hLocal->{hndIn});
if ($iHandleTotal - $self->{oSelect}->count() != 1)
{
confess &log(ASSERT,
"iProcessId $hLocal->{iProcessId}, handle $hLocal->{hndIn} was not removed from select object");
}
# Remove input handle from the map
delete($self->{hLocalMap}{$hLocal->{hndIn}});
# Close the local process
$hLocal->{oLocal}->close(true);
# Undefine local process so it is no longer checked for new jobs
undef(${$self->{hyLocal}}[$iLocalIdx]);
# Skip to next local process
next;
}
# Assign job to local process
$hLocal->{hJob} = $hJob;
$bFound = true;
$self->{iRunning}++;
$self->{iQueued}--;
logDebugMisc(
$strOperation, 'get job from queue',
{name => 'iHostIdx', value => $hLocal->{iHostIdx}},
{name => 'iProcessId', value => $hLocal->{iProcessId}},
{name => 'strQueueIdx', value => $iQueueIdx},
{name => 'strKey', value => $hLocal->{hJob}{strKey}});
# Send job to local process
$hLocal->{oLocal}->cmdWrite($hLocal->{hJob}{strOp}, $hLocal->{hJob}->{rParam});
}
}
# If nothing is running, no more jobs, and nothing to return, then processing is complete
if (!$bFound && !$self->{iRunning} && @hyResult == 0)
{
logDebugMisc($strOperation, 'all jobs complete');
$self->reset();
return;
}
}
# Return job results
return \@hyResult;
}
####################################################################################################################################
# queueJob
#
# Queue a job for processing.
####################################################################################################################################
sub queueJob
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$iHostConfigIdx,
$strQueue,
$strKey,
$strOp,
$rParam,
$rParamSecure,
) =
logDebugParam
(
__PACKAGE__ . '->queueJob', \@_,
{name => 'iHostConfigIdx'},
{name => 'strQueue'},
{name => 'strKey'},
{name => 'strOp'},
{name => 'rParam'},
{name => 'rParamSecure', optional => true, redact => true},
);
# Don't add jobs while in the middle of processing the current queue
if ($self->processing())
{
confess &log(ASSERT, 'new jobs cannot be added until processing is complete');
}
# Copy the parameters to a new variable so we can push the secure parameters on
if (defined($rParamSecure))
{
push(@{$rParam}, @{$rParamSecure});
}
# Build the hash
my $hJob =
{
iHostConfigIdx => $iHostConfigIdx,
strQueue => $strQueue,
strKey => $strKey,
strOp => $strOp,
rParam => $rParam,
};
# Get the host that will perform this job
my $iHostIdx = $self->{hHostMap}{$iHostConfigIdx};
if (!defined($iHostIdx))
{
confess &log(ASSERT, "iHostConfigIdx = $iHostConfigIdx does not exist");
}
my $hHost = $self->{hyHost}[$iHostIdx];
# Get the queue to hold this job
my $iQueueIdx = $hHost->{hQueueMap}{$strQueue};
if (!defined($iQueueIdx))
{
$iQueueIdx = defined($hHost->{hyQueue}) ? @{$hHost->{hyQueue}} : 0;
$hHost->{hQueueMap}{$strQueue} = $iQueueIdx;
}
push(@{$hHost->{hyQueue}[$iQueueIdx]}, $hJob);
$self->{iQueued}++;
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# dequeueJobs
#
# Dequeue all jobs from a queue.
####################################################################################################################################
sub dequeueJobs
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$iHostConfigIdx,
$strQueue,
) =
logDebugParam
(
__PACKAGE__ . '->dequeueJobs', \@_,
{name => 'iHostConfigIdx'},
{name => 'strQueue'},
);
# Don't add jobs while in the middle of processing the current queue
if (!$self->processing())
{
confess &log(ASSERT, 'unable to dequeue a job when not processing');
}
# Get the host that contains the queue to clear
my $iHostIdx = $self->{hHostMap}{$iHostConfigIdx};
if (!defined($iHostIdx))
{
confess &log(ASSERT, "iHostConfigIdx = $iHostConfigIdx does not exist");
}
my $hHost = $self->{hyHost}[$iHostIdx];
# Get the queue to clear
my $iQueueIdx = $hHost->{hQueueMap}{$strQueue};
if (!defined($iQueueIdx))
{
confess &log(ASSERT, "unable to find queue '${strQueue}'");
}
$hHost->{hyQueue}[$iQueueIdx] = [];
$self->{iQueued} = 0;
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# jobTotal
#
# Total jobs in the queue.
####################################################################################################################################
sub jobTotal
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->jobTotal');
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iJobTotal', value => $self->{iQueued} + $self->{iRunning}}
);
}
####################################################################################################################################
# processing
#
# Are jobs being processed?
####################################################################################################################################
sub processing
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->processing');
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bProcessing', value => $self->{bProcessing}, trace => true}
);
}
1;

76
lib/pgBackRest/Protocol/Remote/Master.pm
View File

@ -1,76 +0,0 @@
####################################################################################################################################
# PROTOCOL REMOTE MASTER MODULE
####################################################################################################################################
package pgBackRest::Protocol::Remote::Master;
use parent 'pgBackRest::Protocol::Command::Master';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(dirname);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Command::Master;
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCommandSSH, # SSH client
$strCommand, # Command to execute on local/remote
$iBufferMax, # Maximum buffer size
$iCompressLevel, # Set compression level
$iCompressLevelNetwork, # Set compression level for network only compression
$strHost, # Host to connect to for remote (optional as this can also be used for local)
$strUser, # User to connect to for remote (must be set if strHost is set)
$iSshPort, # Specified if other than default port is needed for ssh
$iProtocolTimeout, # Protocol timeout
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strCommandSSH'},
{name => 'strCommand'},
{name => 'iBufferMax'},
{name => 'iCompressLevel'},
{name => 'iCompressLevelNetwork'},
{name => 'strHost'},
{name => 'strUser'},
{name => 'iSshPort', required => false},
{name => 'iProtocolTimeout'},
);
my $strCommandSshPort = defined($iSshPort) ? '-p ' . $iSshPort . ' ' : '';
# Create SSH command
$strCommand =
"${strCommandSSH} -o LogLevel=error -o Compression=no -o PasswordAuthentication=no $strCommandSshPort" .
"${strUser}\@${strHost} '${strCommand}'";
# Init object and store variables
my $self = $class->SUPER::new(
'remote', "remote process on '$strHost'", $strCommand, $iBufferMax, $iCompressLevel, $iCompressLevelNetwork,
$iProtocolTimeout);
bless $self, $class;
# Store the host
$self->{strHost} = $strHost;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
1;

122
lib/pgBackRest/Protocol/Remote/Minion.pm
View File

@ -1,122 +0,0 @@
####################################################################################################################################
# PROTOCOL REMOTE MINION MODULE
####################################################################################################################################
package pgBackRest::Protocol::Remote::Minion;
use parent 'pgBackRest::Protocol::Command::Minion';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(dirname);
use pgBackRest::Common::Log;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Db;
use pgBackRest::Protocol::Command::Minion;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$iBufferMax, # Maximum buffer size
$iProtocolTimeout # Protocol timeout
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'iBufferMax'},
{name => 'iProtocolTimeout'}
);
# Init object and store variables
my $self = $class->SUPER::new(cfgCommandName(CFGCMD_REMOTE), $iBufferMax, $iProtocolTimeout);
bless $self, $class;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# init
####################################################################################################################################
sub init
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->init');
# Create objects
my $oStorage = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_DB) ? storageDb() : storageRepo();
my $oDb = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_DB) ? new pgBackRest::Db() : undef;
# Create anonymous subs for each command
my $hCommandMap =
{
# Db commands
&OP_DB_CONNECT => sub {$oDb->connect()},
&OP_DB_EXECUTE_SQL => sub {$oDb->executeSql(@{shift()})},
&OP_DB_INFO => sub {$oDb->info(@{shift()})},
# File commands
&OP_STORAGE_OPEN_READ => sub
{
my $oSourceFileIo = $oStorage->openRead(@{shift()});
# If the source file exists
if (defined($oSourceFileIo) && (!defined($oSourceFileIo->{oStorageCRead}) || $oSourceFileIo->open()))
{
$self->outputWrite(true);
$oStorage->copy(
$oSourceFileIo, new pgBackRest::Protocol::Storage::File($self, $oSourceFileIo), {bSourceOpen => true});
return true;
}
return false;
},
&OP_STORAGE_OPEN_WRITE => sub
{
my $oDestinationFileIo = $oStorage->openWrite(@{shift()});
$oStorage->copy(new pgBackRest::Protocol::Storage::File($self, $oDestinationFileIo), $oDestinationFileIo);
},
&OP_STORAGE_CIPHER_PASS_USER => sub {$oStorage->cipherPassUser()},
&OP_STORAGE_EXISTS => sub {$oStorage->exists(@{shift()})},
&OP_STORAGE_LIST => sub {$oStorage->list(@{shift()})},
&OP_STORAGE_MANIFEST => sub {$oStorage->manifestJson(@{shift()})},
&OP_STORAGE_MOVE => sub {$oStorage->move(@{shift()})},
&OP_STORAGE_PATH_GET => sub {$oStorage->pathGet(@{shift()})},
&OP_STORAGE_HASH_SIZE => sub {$oStorage->hashSize(@{shift()})},
# Wait command
&OP_WAIT => sub {waitRemainder(@{shift()})},
};
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hCommandMap', value => $hCommandMap}
);
}
1;

180
lib/pgBackRest/Protocol/Storage/File.pm
View File

@ -1,180 +0,0 @@
####################################################################################################################################
# Protocol File
####################################################################################################################################
package pgBackRest::Protocol::Storage::File;
use parent 'pgBackRest::Common::Io::Base';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub new
{
my $class = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oProtocol, # Master or minion protocol
$oFileIo, # File whose results will be returned via protocol
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oProtocol', trace => true},
{name => 'oFileIo', required => false, trace => true},
);
# Create class
my $self = $class->SUPER::new($oProtocol->io()->id() . ' file');
bless $self, $class;
# Set variables
$self->{oProtocol} = $oProtocol;
$self->{oFileIo} = $oFileIo;
# Set read/write
$self->{bWrite} = false;
# Initialize EOF to false
$self->eofSet(false);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# eof - have reads reached eof?
####################################################################################################################################
sub eof
{
return shift->{bEOF};
}
####################################################################################################################################
# eofSet - set eof
####################################################################################################################################
sub eofSet
{
my $self = shift;
my $bEOF = shift;
$self->{bEOF} = $bEOF;
}
####################################################################################################################################
# read - read block from protocol
####################################################################################################################################
sub read
{
my $self = shift;
my $rtBuffer = shift;
# After EOF always return 0
return 0 if $self->eof();
my $lBlockSize;
# Read the block header and make sure it's valid
my $strBlockHeader = $self->{oProtocol}->io()->readLine();
if ($strBlockHeader !~ /^BRBLOCK[0-9]+$/)
{
confess &log(ERROR, "invalid block header '${strBlockHeader}'", ERROR_FILE_READ);
}
# Get block size from the header
$lBlockSize = substr($strBlockHeader, 7);
# Read block if size > 0
if ($lBlockSize > 0)
{
$self->{oProtocol}->io()->read($rtBuffer, $lBlockSize, true);
}
else
{
$self->eofSet(true);
}
# Return the block size
return $lBlockSize;
}
####################################################################################################################################
# write - write block to protocol
####################################################################################################################################
sub write
{
my $self = shift;
my $rtBuffer = shift;
# Set write
$self->{bWrite} = true;
# Get the buffer size
my $lBlockSize = defined($rtBuffer) ? length($$rtBuffer) : 0;
# Write if size > 0 (0 ends the copy stream so it should only be done in close())
if ($lBlockSize > 0)
{
# Write block header to the protocol stream
$self->{oProtocol}->io()->writeLine("BRBLOCK${lBlockSize}");
# Write block if size
$self->{oProtocol}->io()->write($rtBuffer);
}
return length($$rtBuffer);
}
####################################################################################################################################
# close - set the result hash
####################################################################################################################################
sub close
{
my $self = shift;
# Close if protocol is defined (to prevent this from running more than once)
if (defined($self->{oProtocol}))
{
# If writing output terminator
if ($self->{bWrite})
{
$self->{oProtocol}->io()->writeLine("BRBLOCK0");
}
# On master read the results
if ($self->{oProtocol}->master())
{
($self->{rhResult}) = $self->{oProtocol}->outputRead();
# Minion will send one more output message after file is closed which can be ignored
$self->{oProtocol}->outputRead();
}
# On minion write the results
else
{
$self->{oProtocol}->outputWrite($self->{oFileIo}->resultAll());
}
# Delete protocol to prevent close from running again
delete($self->{oProtocol});
}
}
1;

34
lib/pgBackRest/Protocol/Storage/Helper.pm
View File

@ -14,8 +14,6 @@ use File::Basename qw(basename);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::LibC qw(:storage);
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Remote;
use pgBackRest::Storage::Helper;
####################################################################################################################################
@ -45,35 +43,24 @@ sub storageDb
my
(
$strOperation,
$iRemoteIdx,
) =
logDebugParam
(
__PACKAGE__ . '::storageDb', \@_,
{name => 'iRemoteIdx', optional => true, default => cfgOptionValid(CFGOPT_HOST_ID) ? cfgOption(CFGOPT_HOST_ID) : 1,
trace => true},
);
# Create storage if not defined
if (!defined($hStorage->{&STORAGE_DB}{$iRemoteIdx}))
if (!defined($hStorage->{&STORAGE_DB}))
{
if (isDbLocal({iRemoteIdx => $iRemoteIdx}))
{
$hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Storage::Storage(
STORAGE_DB, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
}
else
{
$hStorage->{&STORAGE_DB}{$iRemoteIdx} = new pgBackRest::Protocol::Storage::Remote(
protocolGet(CFGOPTVAL_REMOTE_TYPE_DB, $iRemoteIdx));
}
$hStorage->{&STORAGE_DB} = new pgBackRest::Storage::Storage(
STORAGE_DB, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oStorageDb', value => $hStorage->{&STORAGE_DB}{$iRemoteIdx}, trace => true},
{name => 'oStorageDb', value => $hStorage->{&STORAGE_DB}, trace => true},
);
}
@ -99,17 +86,8 @@ sub storageRepo
# Create storage if not defined
if (!defined($hStorage->{&STORAGE_REPO}))
{
if (isRepoLocal())
{
$hStorage->{&STORAGE_REPO} = new pgBackRest::Storage::Storage(
STORAGE_REPO, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
}
else
{
# Create remote storage
$hStorage->{&STORAGE_REPO} = new pgBackRest::Protocol::Storage::Remote(
protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP));
}
$hStorage->{&STORAGE_REPO} = new pgBackRest::Storage::Storage(
STORAGE_REPO, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
}
# Return from function and log return values if any

342
lib/pgBackRest/Protocol/Storage/Remote.pm
View File

@ -1,342 +0,0 @@
####################################################################################################################################
# Remote Storage
####################################################################################################################################
package pgBackRest::Protocol::Storage::Remote;
use parent 'pgBackRest::Storage::Base';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::File;
use pgBackRest::Storage::Base;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oProtocol,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oProtocol'},
);
# Init parent
my $self = $class->SUPER::new({lBufferMax => $oProtocol->io()->bufferMax()});
bless $self, $class;
# Set variables
$self->{oProtocol} = $oProtocol;
# Create JSON object
$self->{oJSON} = JSON::PP->new()->allow_nonref();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# exists
####################################################################################################################################
sub exists
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
) =
logDebugParam
(
__PACKAGE__ . '->exists', \@_,
{name => 'strPathExp'},
);
my $bExists = $self->{oProtocol}->cmdExecute(OP_STORAGE_EXISTS, [$strPathExp]);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bExists', value => $bExists}
);
}
####################################################################################################################################
# hashSize
####################################################################################################################################
sub hashSize
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$rhParam,
) =
logDebugParam
(
__PACKAGE__ . '->hashSize', \@_,
{name => 'strPathExp'},
{name => 'rhParam', required => false},
);
my ($strHash, $lSize) = $self->{oProtocol}->cmdExecute(OP_STORAGE_HASH_SIZE, [$strPathExp, $rhParam]);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strHash', value => $strHash},
{name => 'lSize', value => $lSize}
);
}
####################################################################################################################################
# list
####################################################################################################################################
sub list
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$rhParam,
) =
logDebugParam
(
__PACKAGE__ . '->list', \@_,
{name => 'strPathExp'},
{name => 'rhParam', required => false},
);
my @stryFileList = $self->{oProtocol}->cmdExecute(OP_STORAGE_LIST, [$strPathExp, $rhParam]);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'stryFileList', value => \@stryFileList}
);
}
####################################################################################################################################
# manifest
####################################################################################################################################
sub manifest
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$rhParam,
) =
logDebugParam
(
__PACKAGE__ . '->manifest', \@_,
{name => 'strPathExp'},
{name => 'rhParam', required => false},
);
my $hManifest = $self->{oJSON}->decode($self->{oProtocol}->cmdExecute(OP_STORAGE_MANIFEST, [$strPathExp, $rhParam]));
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hManifest', value => $hManifest, trace => true}
);
}
####################################################################################################################################
# openRead
####################################################################################################################################
sub openRead
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFileExp,
$rhParam,
) =
logDebugParam
(
__PACKAGE__ . '->openRead', \@_,
{name => 'strFileExp'},
{name => 'rhParam', required => false},
);
my $oSourceFileIo =
$self->{oProtocol}->cmdExecute(OP_STORAGE_OPEN_READ, [$strFileExp, $rhParam]) ?
new pgBackRest::Protocol::Storage::File($self->{oProtocol}) : undef;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oFileIo', value => $oSourceFileIo, trace => true},
);
}
####################################################################################################################################
# openWrite
####################################################################################################################################
sub openWrite
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strFileExp,
$rhParam,
) =
logDebugParam
(
__PACKAGE__ . '->openWrite', \@_,
{name => 'strFileExp'},
{name => 'rhParam', required => false},
);
# Open the remote file
$self->{oProtocol}->cmdWrite(OP_STORAGE_OPEN_WRITE, [$strFileExp, $rhParam]);
my $oDestinationFileIo = new pgBackRest::Protocol::Storage::File($self->{oProtocol});
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oFileIo', value => $oDestinationFileIo, trace => true},
);
}
####################################################################################################################################
# pathGet
####################################################################################################################################
sub pathGet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$rhParam,
) =
logDebugParam
(
__PACKAGE__ . '->pathGet', \@_,
{name => 'strPathExp', required => false},
{name => 'rhParam', required => false},
);
my $strPath = $self->{oProtocol}->cmdExecute(OP_STORAGE_PATH_GET, [$strPathExp, $rhParam]);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strPath', value => $strPath}
);
}
####################################################################################################################################
# move
####################################################################################################################################
sub move
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strSourcePathExp,
$strDestinationPathExp,
$rhParam,
) =
logDebugParam
(
__PACKAGE__ . '->move', \@_,
{name => 'strSourcePathExp'},
{name => 'strDestinationPathExp'},
{name => 'rhParam', required => false},
);
$self->{oProtocol}->cmdExecute(OP_STORAGE_MOVE, [$strSourcePathExp, $strDestinationPathExp, $rhParam], false);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation
);
}
####################################################################################################################################
# cipherPassUser
####################################################################################################################################
sub cipherPassUser
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
) =
logDebugParam
(
__PACKAGE__ . '->cipherPassUser', \@_,
);
my $strCipherPassUser = $self->{oProtocol}->cmdExecute(OP_STORAGE_CIPHER_PASS_USER);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strCipherPassUser', value => $strCipherPassUser, redact => true}
);
}
####################################################################################################################################
# getters
####################################################################################################################################
sub protocol {shift->{oProtocol}};
1;

2
libc/LibC.xs
View File

@ -56,7 +56,6 @@ These includes are from the src directory. There is no Perl-specific code in th
#include "config/define.h"
#include "config/load.h"
#include "config/parse.h"
#include "perl/config.h"
#include "postgres/pageChecksum.h"
#include "storage/posix/storage.h"
@ -72,6 +71,7 @@ These includes define data structures that are required for the C to Perl interf
***********************************************************************************************************************************/
#include "xs/crypto/hash.xsh"
#include "xs/common/encode.xsh"
#include "xs/config/configTest.xsh"
#include "xs/postgres/client.xsh"
#include "xs/storage/storage.xsh"
#include "xs/storage/storageRead.xsh"

1
libc/Makefile.PL
View File

@ -93,7 +93,6 @@ my @stryCFile =
'config/define.c',
'config/load.c',
'config/parse.c',
'perl/config.c',
'protocol/client.c',
'protocol/command.c',
'protocol/helper.c',

9
src/perl/config.c → libc/xs/config/configTest.xsh
View File

@ -1,11 +1,8 @@
/***********************************************************************************************************************************
Perl Configuration
Config XS Header
***********************************************************************************************************************************/
#include "build.auto.h"
#include <string.h>
#include "common/debug.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "config/config.h"
@ -16,8 +13,6 @@ Build JSON output from options
String *
perlOptionJson(void)
{
FUNCTION_TEST_VOID();
String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
@ -121,5 +116,5 @@ perlOptionJson(void)
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN(result);
return result;
}

2
libc/xs/storage/storage.xs
View File

@ -22,7 +22,6 @@ CODE:
memContextSwitch(MEM_CONTEXT_XS_OLD());
RETVAL = storagePosixNew(
path == NULL ? STRDEF("/") : path, STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
storagePathEnforceSet((Storage *)RETVAL, false);
memContextSwitch(MEM_CONTEXT_XS_TEMP());
}
else if (strEqZ(type, "<REPO>"))
@ -36,7 +35,6 @@ CODE:
memContextSwitch(MEM_CONTEXT_XS_OLD());
RETVAL = storagePosixNew(cfgOptionStr(cfgOptPgPath), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
storagePathEnforceSet((Storage *)RETVAL, false);
memContextSwitch(MEM_CONTEXT_XS_TEMP());
}
else

18
src/Makefile.in
View File

@ -8,7 +8,7 @@
CC = @CC@
# Standards
CSTANDARD = -std=c99 -D_POSIX_C_SOURCE=200112L -D_DARWIN_C_SOURCE
CSTANDARD = -std=c99 -D_POSIX_C_SOURCE=200809L -D_DARWIN_C_SOURCE
# Optimizations
COPTIMIZE = @COPTIMIZE@
@ -140,8 +140,6 @@ SRCS = \
info/infoBackup.c \
info/manifest.c \
info/infoPg.c \
perl/config.c \
perl/exec.c \
postgres/client.c \
postgres/interface.c \
postgres/interface/v083.c \
@ -220,7 +218,7 @@ command/archive/common.o: command/archive/common.c build.auto.h command/archive/
command/archive/get/file.o: command/archive/get/file.c build.auto.h command/archive/common.h command/archive/get/file.h command/control/common.h common/assert.h common/compress/gzip/common.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoPg.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/get/file.c -o command/archive/get/file.o
command/archive/get/get.o: command/archive/get/get.c build.auto.h command/archive/common.h command/archive/get/file.h command/archive/get/protocol.h command/command.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h perl/exec.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
command/archive/get/get.o: command/archive/get/get.c build.auto.h command/archive/common.h command/archive/get/file.h command/archive/get/protocol.h command/command.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/get/get.c -o command/archive/get/get.o
command/archive/get/protocol.o: command/archive/get/protocol.c build.auto.h command/archive/get/file.h command/archive/get/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
@ -274,7 +272,7 @@ command/expire/expire.o: command/expire/expire.c build.auto.h command/archive/co
command/help/help.o: command/help/help.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/help/help.c -o command/help/help.o
command/info/info.o: command/info/info.c build.auto.h command/archive/common.h command/backup/common.h command/info/info.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
command/info/info.o: command/info/info.c build.auto.h command/archive/common.h command/backup/common.h command/info/info.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/info/info.c -o command/info/info.o
command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h command/backup/protocol.h command/restore/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
@ -340,7 +338,7 @@ common/error.o: common/error.c build.auto.h common/error.auto.c common/error.aut
common/exec.o: common/exec.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exec.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/exec.c -o common/exec.o
common/exit.o: common/exit.c build.auto.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h perl/exec.h protocol/client.h protocol/command.h protocol/helper.h
common/exit.o: common/exit.c build.auto.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/client.h protocol/command.h protocol/helper.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/exit.c -o common/exit.o
common/fork.o: common/fork.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/stackTrace.h common/type/convert.h common/type/stringz.h
@ -502,15 +500,9 @@ info/infoPg.o: info/infoPg.c build.auto.h common/assert.h common/crypto/common.h
info/manifest.o: info/manifest.c build.auto.h command/backup/common.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/mcv.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h info/info.h info/manifest.h postgres/interface.h postgres/version.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/manifest.c -o info/manifest.o
main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/backup/backup.h command/check/check.h command/command.h command/control/start.h command/control/stop.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/restore/restore.h command/stanza/create.h command/stanza/delete.h command/stanza/upgrade.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h
main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/backup/backup.h command/check/check.h command/command.h command/control/start.h command/control/stop.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/restore/restore.h command/stanza/create.h command/stanza/delete.h command/stanza/upgrade.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c main.c -o main.o
perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/config.c -o perl/config.o
perl/exec.o: perl/exec.c ../libc/LibC.h build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/encode.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h perl/config.h perl/embed.auto.c perl/exec.h perl/libc.auto.c postgres/client.h postgres/interface.h postgres/pageChecksum.h storage/helper.h storage/info.h storage/posix/storage.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h ../libc/xs/common/encode.xsh ../libc/xs/crypto/hash.xsh ../libc/xs/postgres/client.xsh ../libc/xs/storage/storage.xsh ../libc/xs/storage/storageRead.xsh ../libc/xs/storage/storageWrite.xsh
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c perl/exec.c -o perl/exec.o
postgres/client.o: postgres/client.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h postgres/client.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c postgres/client.c -o postgres/client.o

3
src/build.auto.h.in
View File

@ -3,6 +3,3 @@ Build Flags Generated by Configure
***********************************************************************************************************************************/
// Are test code and asserts disabled?
#undef NDEBUG
// Is libperl present?
#undef HAVE_LIBPERL

1
src/command/archive/get/get.c
View File

@ -21,7 +21,6 @@ Archive Get Command
#include "common/wait.h"
#include "config/config.h"
#include "config/exec.h"
#include "perl/exec.h"
#include "postgres/interface.h"
#include "protocol/helper.h"
#include "protocol/parallel.h"

4
src/command/backup/pageChecksum.c
View File

@ -146,8 +146,8 @@ pageChecksumResult(THIS_VOID)
VariantList *errorList = varLstNew();
unsigned int errorIdx = 0;
// Convert the full list to an abbreviated list that the Perl code can understand. In the future we want to return the
// entire list so pages can be verified in the WAL.
// Convert the full list to an abbreviated list. In the future we want to return the entire list so pages can be verified
// in the WAL.
do
{
unsigned int pageId = varUInt(varLstGet(varVarLst(varLstGet(this->error, errorIdx)), 0));

1
src/command/info/info.c
View File

@ -22,7 +22,6 @@ Info Command
#include "info/infoBackup.h"
#include "info/infoPg.h"
#include "info/manifest.h"
#include "perl/exec.h"
#include "postgres/interface.h"
#include "storage/helper.h"

47
src/common/exit.c
View File

@ -15,10 +15,6 @@ Exit Routines
#include "config/config.h"
#include "protocol/helper.h"
#ifdef HAVE_LIBPERL
#include "perl/exec.h"
#endif
/***********************************************************************************************************************************
Return signal names
***********************************************************************************************************************************/
@ -106,32 +102,24 @@ exitSafe(int result, bool error, SignalType signalType)
// Report error if one was thrown
if (error)
{
// Don't log the error if it has already been logged by Perl
#ifdef HAVE_LIBPERL
if (strcmp(errorMessage(), PERL_EMBED_ERROR) != 0)
LogLevel logLevel = errorCode() == errorTypeCode(&AssertError) ? logLevelAssert : logLevelError;
// Assert errors always output a stack trace
if (logLevel == logLevelAssert)
LOG_FMT(logLevel, errorCode(), "%s\nSTACK TRACE:\n%s", errorMessage(), errorStackTrace());
else
{
#endif
LogLevel logLevel = errorCode() == errorTypeCode(&AssertError) ? logLevelAssert : logLevelError;
// Log just the error to non-debug levels
LOG_INTERNAL(logLevel, LOG_LEVEL_MIN, logLevelDetail, 0, errorCode(), errorMessage());
// Assert errors always output a stack trace
if (logLevel == logLevelAssert)
LOG_FMT(logLevel, errorCode(), "%s\nSTACK TRACE:\n%s", errorMessage(), errorStackTrace());
else
// Log the stack trace debug levels
if (logAny(logLevelDebug))
{
// Log just the error to non-debug levels
LOG_INTERNAL(logLevel, LOG_LEVEL_MIN, logLevelDetail, 0, errorCode(), errorMessage());
// Log the stack trace debug levels
if (logAny(logLevelDebug))
{
LOG_INTERNAL_FMT(
logLevel, logLevelDebug, LOG_LEVEL_MAX, 0, errorCode(), "%s\nSTACK TRACE:\n%s", errorMessage(),
errorStackTrace());
}
LOG_INTERNAL_FMT(
logLevel, logLevelDebug, LOG_LEVEL_MAX, 0, errorCode(), "%s\nSTACK TRACE:\n%s", errorMessage(),
errorStackTrace());
}
#ifdef HAVE_LIBPERL
}
#endif
result = errorCode();
}
@ -143,15 +131,6 @@ exitSafe(int result, bool error, SignalType signalType)
}
TRY_END();
// Free Perl but ignore errors
#ifdef HAVE_LIBPERL
TRY_BEGIN()
{
perlFree(result);
}
TRY_END();
#endif
// Log command end if a command is set
if (cfgCommand() != cfgCmdNone)
{

9
src/config/config.auto.c
View File

@ -310,7 +310,6 @@ STRING_EXTERN(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX_STR, CFGOPT_ARCHI
STRING_EXTERN(CFGOPT_ARCHIVE_TIMEOUT_STR, CFGOPT_ARCHIVE_TIMEOUT);
STRING_EXTERN(CFGOPT_BACKUP_STANDBY_STR, CFGOPT_BACKUP_STANDBY);
STRING_EXTERN(CFGOPT_BUFFER_SIZE_STR, CFGOPT_BUFFER_SIZE);
STRING_EXTERN(CFGOPT_C_STR, CFGOPT_C);
STRING_EXTERN(CFGOPT_CHECKSUM_PAGE_STR, CFGOPT_CHECKSUM_PAGE);
STRING_EXTERN(CFGOPT_CMD_SSH_STR, CFGOPT_CMD_SSH);
STRING_EXTERN(CFGOPT_COMMAND_STR, CFGOPT_COMMAND);
@ -536,14 +535,6 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptBufferSize)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
CONFIG_OPTION_NAME(CFGOPT_C)
CONFIG_OPTION_INDEX(0)
CONFIG_OPTION_DEFINE_ID(cfgDefOptC)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(

5
src/config/config.auto.h
View File

@ -69,8 +69,6 @@ Option constants
STRING_DECLARE(CFGOPT_BACKUP_STANDBY_STR);
#define CFGOPT_BUFFER_SIZE "buffer-size"
STRING_DECLARE(CFGOPT_BUFFER_SIZE_STR);
#define CFGOPT_C "c"
STRING_DECLARE(CFGOPT_C_STR);
#define CFGOPT_CHECKSUM_PAGE "checksum-page"
STRING_DECLARE(CFGOPT_CHECKSUM_PAGE_STR);
#define CFGOPT_CMD_SSH "cmd-ssh"
@ -382,7 +380,7 @@ Option constants
#define CFGOPT_TYPE "type"
STRING_DECLARE(CFGOPT_TYPE_STR);
#define CFG_OPTION_TOTAL 164
#define CFG_OPTION_TOTAL 163
/***********************************************************************************************************************************
Command enum
@ -424,7 +422,6 @@ typedef enum
cfgOptArchiveTimeout,
cfgOptBackupStandby,
cfgOptBufferSize,
cfgOptC,
cfgOptChecksumPage,
cfgOptCmdSsh,
cfgOptCommand,

2
src/config/config.h
View File

@ -86,7 +86,7 @@ typedef enum
Load Functions
Used primarily by modules that need to manipulate the configuration. These modules include, but are not limited to, config/parse.c,
config/load.c, perl/config.c, and perl/exec.c.
config/load.c.
***********************************************************************************************************************************/
void cfgInit(void);

23
src/config/define.auto.c
View File

@ -582,29 +582,6 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
CFGDEFDATA_OPTION_NAME("c")
CFGDEFDATA_OPTION_REQUIRED(true)
CFGDEFDATA_OPTION_SECTION(cfgDefSectionCommandLine)
CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeBoolean)
CFGDEFDATA_OPTION_INTERNAL(true)
CFGDEFDATA_OPTION_INDEX_TOTAL(1)
CFGDEFDATA_OPTION_SECURE(false)
CFGDEFDATA_OPTION_COMMAND_LIST
(
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRemote)
)
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("0")
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(

1
src/config/define.auto.h
View File

@ -60,7 +60,6 @@ typedef enum
cfgDefOptArchiveTimeout,
cfgDefOptBackupStandby,
cfgDefOptBufferSize,
cfgDefOptC,
cfgDefOptChecksumPage,
cfgDefOptCmdSsh,
cfgDefOptCommand,

3
src/config/load.c
View File

@ -236,8 +236,7 @@ cfgLoadUpdateOption(void)
}
/***********************************************************************************************************************************
Attempt to set the log file and turn file logging off if the file cannot be opened. This is so the Perl code won't attempt to open
the file again and error out.
Attempt to set the log file and turn file logging off if the file cannot be opened
***********************************************************************************************************************************/
void
cfgLoadLogFile(void)

8
src/config/parse.auto.c
View File

@ -126,13 +126,6 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptBufferSize,
},
// c option
// -----------------------------------------------------------------------------------------------------------------------------
{
.name = CFGOPT_C,
.val = PARSE_OPTION_FLAG | cfgOptC,
},
// checksum-page option
// -----------------------------------------------------------------------------------------------------------------------------
{
@ -2253,7 +2246,6 @@ static const ConfigOption optionResolveOrder[] =
cfgOptArchiveTimeout,
cfgOptBackupStandby,
cfgOptBufferSize,
cfgOptC,
cfgOptChecksumPage,
cfgOptCmdSsh,
cfgOptCommand,

55
src/configure vendored
View File

@ -2807,61 +2807,8 @@ fi
CINCLUDE="-I. -I../libc"
# Check required perl library
LIBS_BEFORE_PERL=$LIBS
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for perl_parse in -lperl" >&5
$as_echo_n "checking for perl_parse in -lperl... " >&6; }
if ${ac_cv_lib_perl_perl_parse+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
LIBS="-lperl `perl -MExtUtils::Embed -e ldopts` $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
#ifdef __cplusplus
extern "C"
#endif
char perl_parse ();
int
main ()
{
return perl_parse ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
ac_cv_lib_perl_perl_parse=yes
else
ac_cv_lib_perl_perl_parse=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_perl_perl_parse" >&5
$as_echo "$ac_cv_lib_perl_perl_parse" >&6; }
if test "x$ac_cv_lib_perl_perl_parse" = xyes; then :
cat >>confdefs.h <<_ACEOF
#define HAVE_LIBPERL 1
_ACEOF
LIBS="-lperl $LIBS"
else
as_fn_error $? "library 'perl' is required" "$LINENO" 5
fi
LIBS="$LIBS_BEFORE_PERL `perl -MExtUtils::Embed -e ldopts`"
CLIBRARY="`perl -MExtUtils::Embed -e ccopts`"
# Check required pq library
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for PQconnectdb in -lpq" >&5
$as_echo_n "checking for PQconnectdb in -lpq... " >&6; }
if ${ac_cv_lib_pq_PQconnectdb+:} false; then :

6
src/configure.ac
View File

@ -33,12 +33,6 @@ fi
# Set includes
AC_SUBST(CINCLUDE, "-I. -I../libc")
# Check required perl library
LIBS_BEFORE_PERL=$LIBS
AC_CHECK_LIB([perl], [perl_parse], [], [AC_MSG_ERROR([library 'perl' is required])], [`perl -MExtUtils::Embed -e ldopts`])
LIBS="$LIBS_BEFORE_PERL `perl -MExtUtils::Embed -e ldopts`"
AC_SUBST(CLIBRARY, "`perl -MExtUtils::Embed -e ccopts`")
# Check required pq library
AC_CHECK_LIB([pq], [PQconnectdb], [], [AC_MSG_ERROR([library 'pq' is required])])
AC_SUBST(CINCLUDE, "$CINCLUDE -I`pg_config --includedir`")

4
src/info/infoPg.c
View File

@ -297,7 +297,7 @@ infoPgSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveData
InfoPgData pgData = infoPgDataCurrent(saveData->infoPg);
// These need to be saved because older Perl and C versions expect them
// These need to be saved because older pgBackRest versions expect them
if (saveData->infoPg->type == infoPgBackup)
{
infoSaveValue(
@ -331,7 +331,7 @@ infoPgSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveData
{
kvPut(pgDataKv, INFO_KEY_DB_SYSTEM_ID_VAR, VARUINT64(pgData.systemId));
// These need to be saved because older Perl and C versions expect them
// These need to be saved because older pgBackRest versions expect them
kvPut(pgDataKv, INFO_KEY_DB_CATALOG_VERSION_VAR, VARUINT(pgCatalogVersion(pgData.version)));
kvPut(pgDataKv, INFO_KEY_DB_CONTROL_VERSION_VAR, VARUINT(pgControlVersion(pgData.version)));
}

9
src/main.c
View File

@ -30,7 +30,6 @@ Main
#include "config/config.h"
#include "config/load.h"
#include "postgres/interface.h"
#include "perl/exec.h"
#include "storage/helper.h"
#include "version.h"
@ -166,13 +165,7 @@ main(int argListSize, const char *argList[])
// -----------------------------------------------------------------------------------------------------------------
case cfgCmdRemote:
{
if (cfgOptionBool(cfgOptC))
{
cmdRemote(STDIN_FILENO, STDOUT_FILENO);
}
else
perlExec();
cmdRemote(STDIN_FILENO, STDOUT_FILENO);
break;
}

14
src/perl/config.h
View File

@ -1,14 +0,0 @@
/***********************************************************************************************************************************
Perl Configuration
***********************************************************************************************************************************/
#ifndef PERL_CONFIG_H
#define PERL_CONFIG_H
#include "common/type/string.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
String *perlOptionJson(void);
#endif

11537
src/perl/embed.auto.c
View File

File diff suppressed because it is too large Load Diff

276
src/perl/exec.c
View File

@ -1,276 +0,0 @@
/***********************************************************************************************************************************
Execute Perl for Legacy Functionality
***********************************************************************************************************************************/
#include "build.auto.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "version.h"
#include "common/debug.h"
#include "common/error.h"
#include "common/memContext.h"
#include "config/config.h"
#include "perl/config.h"
#include "perl/exec.h"
/***********************************************************************************************************************************
Include LibC code
This file is generated by the LibC xs build. Including it here allows the functions provided by the C library to be provided by the
pgBackRest binary instead which means the C library does not need to be deployed in production builds.
***********************************************************************************************************************************/
#ifndef HAS_BOOL
# define HAS_BOOL 1
#endif
#include "perl/libc.auto.c"
/***********************************************************************************************************************************
Include embedded Perl modules
***********************************************************************************************************************************/
typedef struct EmbeddedModule
{
const char *name;
const char *data;
} EmbeddedModule;
#include "perl/embed.auto.c"
/***********************************************************************************************************************************
Perl interpreter
This is a silly name but Perl prefers it.
***********************************************************************************************************************************/
static PerlInterpreter *my_perl = NULL;
/***********************************************************************************************************************************
Constants used to build perl options
***********************************************************************************************************************************/
#define PGBACKREST_MODULE PROJECT_NAME "::Main"
#define PGBACKREST_MAIN PGBACKREST_MODULE "::main"
/***********************************************************************************************************************************
Build list of parameters to use for perl main
***********************************************************************************************************************************/
String *
perlMain(void)
{
FUNCTION_TEST_VOID();
// Add command arguments to pass to main
String *commandParam = strNew("");
for (unsigned int paramIdx = 0; paramIdx < strLstSize(cfgCommandParam()); paramIdx++)
strCatFmt(commandParam, ",'%s'", strPtr(strLstGet(cfgCommandParam(), paramIdx)));
// Construct Perl main call
String *mainCall = strNewFmt(
"($iResult, $bErrorC, $strMessage) = " PGBACKREST_MAIN "('%s'%s)", cfgCommandName(cfgCommand()), strPtr(commandParam));
FUNCTION_TEST_RETURN(mainCall);
}
/***********************************************************************************************************************************
Dynamic module loader
***********************************************************************************************************************************/
static const char *
embeddedModuleGetInternal(const char *moduleName)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRINGZ, moduleName);
FUNCTION_TEST_END();
// Find module
const char *result = NULL;
for (unsigned int moduleIdx = 0; moduleIdx < sizeof(embeddedModule) / sizeof(EmbeddedModule); moduleIdx++)
{
if (strcmp(embeddedModule[moduleIdx].name, moduleName) == 0)
{
result = embeddedModule[moduleIdx].data;
break;
}
}
// Error if the module was not found
if (result == NULL)
THROW_FMT(AssertError, "unable to load embedded module '%s'", moduleName);
FUNCTION_TEST_RETURN(result);
}
XS_EUPXS(embeddedModuleGet);
XS_EUPXS(embeddedModuleGet)
{
dVAR; dXSARGS;
(void)cv;
// Get module name
CHECK(items == 1);
const char *moduleName = (const char *)SvPV_nolen(ST(0)); // {uncoverable_branch - Perl macro}
dXSTARG; // {uncoverable_branch - Perl macro}
// Return module data
sv_setpv(TARG, embeddedModuleGetInternal(moduleName));
XSprePUSH;
PUSHTARG; // {uncoverable_branch - Perl macro}
XSRETURN(1);
}
/***********************************************************************************************************************************
Init the dynaloader so other C modules can be loaded
There are no FUNCTION_TEST* calls because this is a callback from Perl and it doesn't seem wise to mix our stack stuff up in it.
***********************************************************************************************************************************/
#define LOADER_SUB \
"sub\n" \
"{\n" \
" if ($_[1] =~ /^pgBackRest/)\n" \
" {\n" \
" my $data = pgBackRest::LibC::embeddedModuleGet($_[1]);\n" \
"\n" \
" open my $fh, '<', \\$data;\n" \
" return $fh;\n" \
" }\n" \
"}"
EXTERN_C void boot_DynaLoader (pTHX_ CV* cv);
static void xs_init(pTHX)
{
dXSUB_SYS;
PERL_UNUSED_CONTEXT;
// Register the LibC functions by registering the boot function and calling it
newXS("pgBackRest::LibC::boot", boot_pgBackRest__LibC, __FILE__);
eval_pv("pgBackRest::LibC::boot()", TRUE);
// Register the embedded module getter
newXS("pgBackRest::LibC::embeddedModuleGet", embeddedModuleGet, __FILE__);
// DynaLoader is a special case
newXS("DynaLoader::boot_DynaLoader", boot_DynaLoader, __FILE__);
}
/***********************************************************************************************************************************
Evaluate a perl statement
***********************************************************************************************************************************/
static void
perlEval(const String *statement)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, statement);
FUNCTION_TEST_END();
eval_pv(strPtr(statement), TRUE);
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
Initialize Perl
***********************************************************************************************************************************/
static void
perlInit(void)
{
FUNCTION_TEST_VOID();
if (!my_perl)
{
// Initialize Perl with dummy args and environment
int argc = 1;
const char *argv[1] = {strPtr(cfgExe())};
const char *env[1] = {NULL};
PERL_SYS_INIT3(&argc, (char ***)&argv, (char ***)&env);
// Create the interpreter
const char *embedding[] = {"", "-e", "0"};
my_perl = perl_alloc();
perl_construct(my_perl);
// Don't let $0 assignment update the proctitle or embedding[0]
PL_origalen = 1;
// Start the interpreter
perl_parse(my_perl, xs_init, 3, (char **)embedding, NULL);
PL_exit_flags |= PERL_EXIT_DESTRUCT_END;
perl_run(my_perl);
// Use customer loader to get all embedded modules
eval_pv("splice(@INC, 0, 0, " LOADER_SUB ");", true);
// Now that the custom loader is installed, load the main module;
eval_pv("use " PGBACKREST_MODULE ";", true);
// Set config data -- this is done separately to avoid it being included in stack traces
perlEval(strNewFmt(PGBACKREST_MAIN "ConfigSet('%s', '%s')", strPtr(cfgExe()), strPtr(perlOptionJson())));
}
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
Execute main function in Perl
***********************************************************************************************************************************/
static int perlExecResult(int code, bool errorC, const char *message)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INT, code);
FUNCTION_TEST_PARAM(BOOL, errorC);
FUNCTION_TEST_PARAM(STRINGZ, message);
FUNCTION_TEST_END();
int result = code;
if (code >= errorTypeCode(&AssertError))
{
if (errorC)
RETHROW();
else
THROW_CODE(code, strlen(message) == 0 ? PERL_EMBED_ERROR : message);
}
FUNCTION_TEST_RETURN(result);
}
int
perlExec(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
// Initialize Perl
perlInit();
// Run perl main function
perlEval(perlMain());
// Return result code
int code = (int)SvIV(get_sv("iResult", 0)); // {uncoverable_branch - Perl macro}
bool errorC = (int)SvIV(get_sv("bErrorC", 0)); // {uncoverable_branch - Perl macro}
char *message = SvPV_nolen(get_sv("strMessage", 0)); // {uncoverable_branch - Perl macro}
FUNCTION_LOG_RETURN(INT, perlExecResult(code, errorC, message));
}
/***********************************************************************************************************************************
Free Perl objects
Don't bother freeing Perl itself since we are about to exit.
***********************************************************************************************************************************/
void
perlFree(int result)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INT, result);
FUNCTION_TEST_END();
if (my_perl != NULL)
perlEval(strNewFmt(PGBACKREST_MAIN "Cleanup(%d)", result));
FUNCTION_TEST_RETURN_VOID();
}

18
src/perl/exec.h
View File

@ -1,18 +0,0 @@
/***********************************************************************************************************************************
Execute Perl for Legacy Functionality
***********************************************************************************************************************************/
#ifndef PERL_EXEC_H
#define PERL_EXEC_H
/***********************************************************************************************************************************
Error message for errors thrown from embedded Perl
***********************************************************************************************************************************/
#define PERL_EMBED_ERROR "===PERL-EMBED-ERROR==="
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
int perlExec(void);
void perlFree(int result);
#endif

2337
src/perl/libc.auto.c
View File

File diff suppressed because it is too large Load Diff

3
src/protocol/helper.c
View File

@ -265,9 +265,6 @@ protocolRemoteParam(ProtocolStorageType protocolStorageType, unsigned int protoc
optionReplace, VARSTR(CFGOPT_CONFIG_PATH_STR),
cfgOptionSource(optConfigPath) != cfgSourceDefault ? cfgOption(optConfigPath) : NULL);
// Use a C remote
kvPut(optionReplace, VARSTR(CFGOPT_C_STR), VARBOOL(true));
// Copy pg options to index 0 since that's what the remote will be expecting
if (hostIdx != 0)
{

20
src/storage/storage.c
View File

@ -30,7 +30,6 @@ struct Storage
mode_t modeFile;
mode_t modePath;
bool write;
bool pathEnforce;
StoragePathExpressionCallback *pathExpressionFunction;
};
@ -75,7 +74,6 @@ storageNew(
this->path = strDup(path);
this->modeFile = modeFile;
this->modePath = modePath;
this->pathEnforce = true;
this->write = write;
this->pathExpressionFunction = pathExpressionFunction;
@ -681,7 +679,7 @@ storagePath(const Storage *this, const String *pathExp, StoragePathParam param)
// Make sure the base storage path is contained within the path expression
if (!strEqZ(this->path, "/"))
{
if (this->pathEnforce && !param.noEnforce && (!strBeginsWith(pathExp, this->path) ||
if (!param.noEnforce && (!strBeginsWith(pathExp, this->path) ||
!(strSize(pathExp) == strSize(this->path) || *(strPtr(pathExp) + strSize(this->path)) == '/')))
{
THROW_FMT(AssertError, "absolute path '%s' is not in base path '%s'", strPtr(pathExp), strPtr(this->path));
@ -966,22 +964,6 @@ storageInterface(const Storage *this)
FUNCTION_TEST_RETURN(this->interface);
}
/***********************************************************************************************************************************
Set whether absolute paths are required to be in the base path
***********************************************************************************************************************************/
void
storagePathEnforceSet(Storage *this, bool enforce)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE, this);
FUNCTION_TEST_PARAM(BOOL, enforce);
FUNCTION_TEST_END();
this->pathEnforce = enforce;
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
Get the storage type (posix, cifs, etc.)
***********************************************************************************************************************************/

4
src/storage/storage.intern.h
View File

@ -302,10 +302,6 @@ Getters/Setters
void *storageDriver(const Storage *this);
StorageInterface storageInterface(const Storage *this);
// The option is intended to be used only with the Perl interface since Perl is not tidy about where it reads. It should be
// removed when the Perl interface is removed.
void storagePathEnforceSet(Storage *this, bool enforce);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/

97
test/define.yaml
View File

@ -16,7 +16,7 @@
# * define - defines for C code (will also be applied to the test harness)
# * define-test - defines for the test harness
# * debugUnitSuppress - don't define DEBUG_UNIT for unit tests -- this is used to test unit test debugging macros
# * perlReq - is Perl required for this C test?
# * binReq - is the pgbackrest binary required for this test?
# * containerReq - is this test required to run in a container?
#
# Some options are unique to tests:
@ -28,8 +28,6 @@
# **********************************************************************************************************************************
# Unit tests
#
# Unit tests are assumed to be C tests unless they end in "-perl".
# **********************************************************************************************************************************
unit:
@ -291,10 +289,6 @@ unit:
common/encode: full
common/encode/base64: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: encode-perl
total: 1
# ----------------------------------------------------------------------------------------------------------------------------
- name: reg-exp
total: 3
@ -309,26 +303,6 @@ unit:
coverage:
common/ini: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: ini-perl
total: 10
# ----------------------------------------------------------------------------------------------------------------------------
- name: io-handle-perl
total: 6
# ----------------------------------------------------------------------------------------------------------------------------
- name: io-buffered-perl
total: 3
# ----------------------------------------------------------------------------------------------------------------------------
- name: io-process-perl
total: 3
# ----------------------------------------------------------------------------------------------------------------------------
- name: log-perl
total: 1
# ********************************************************************************************************************************
- name: postgres
@ -354,26 +328,6 @@ unit:
coverage:
postgres/pageChecksum: full
# ********************************************************************************************************************************
- name: perl
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: config
total: 1
coverage:
perl/config: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: exec
total: 4
containerReq: true
perlReq: true
coverage:
perl/exec: full
# ********************************************************************************************************************************
- name: config
@ -427,17 +381,6 @@ unit:
- name: storage
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: perl
total: 13
coverage:
Storage/Storage: partial
# ----------------------------------------------------------------------------------------------------------------------------
- name: helper-perl
total: 3
# ----------------------------------------------------------------------------------------------------------------------------
- name: cifs
total: 1
@ -468,7 +411,7 @@ unit:
- name: remote
total: 12
containerReq: true
perlReq: true
binReq: true
coverage:
storage/remote/read: full
@ -497,19 +440,11 @@ unit:
- name: protocol
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: common-minion-perl
total: 1
# ----------------------------------------------------------------------------------------------------------------------------
- name: helper-perl
total: 2
# ----------------------------------------------------------------------------------------------------------------------------
- name: protocol
total: 9
containerReq: true
perlReq: true
binReq: true
coverage:
protocol/client: full
@ -544,10 +479,6 @@ unit:
coverage:
info/infoArchive: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: info-archive-perl
total: 4
# ----------------------------------------------------------------------------------------------------------------------------
- name: info-backup
total: 2
@ -562,10 +493,6 @@ unit:
coverage:
info/manifest: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: info-backup-perl
total: 3
# ********************************************************************************************************************************
- name: db
@ -574,7 +501,7 @@ unit:
- name: db
total: 3
containerReq: true
perlReq: true
binReq: true
coverage:
db/db: full
@ -595,7 +522,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-get
total: 5
perlReq: true
binReq: true
coverage:
command/archive/get/file: full
@ -605,7 +532,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-push
total: 4
perlReq: true
binReq: true
coverage:
command/archive/push/file: full
@ -623,7 +550,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: backup
total: 10
perlReq: true
binReq: true
coverage:
command/backup/backup: full
@ -698,7 +625,7 @@ unit:
- name: restore
total: 11
containerReq: true
perlReq: true
binReq: true
coverage:
command/restore/file: full
@ -727,14 +654,6 @@ unit:
coverage:
command/storage/list: full
# ********************************************************************************************************************************
- name: manifest
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: all-perl
total: 11
# **********************************************************************************************************************************
# Integration tests
#

6
test/lib/pgBackRestTest/Common/DefineTest.pm
View File

@ -63,8 +63,8 @@ use constant TESTDEF_TOTAL => 'total';
push @EXPORT, qw(TESTDEF_TOTAL);
use constant TESTDEF_TYPE => 'type';
push @EXPORT, qw(TESTDEF_TYPE);
use constant TESTDEF_PERL_REQ => 'perlReq';
push @EXPORT, qw(TESTDEF_PERL_REQ);
use constant TESTDEF_BIN_REQ => 'binReq';
push @EXPORT, qw(TESTDEF_BIN_REQ);
use constant TESTDEF_VM => 'vm';
push @EXPORT, qw(TESTDEF_VM);
@ -128,7 +128,7 @@ sub testDefLoad
# Resolve variables that can be set in the module or the test
foreach my $strVar (
TESTDEF_DEFINE, TESTDEF_DEFINE_TEST, TESTDEF_DEBUG_UNIT_SUPPRESS, TESTDEF_DB, TESTDEF_PERL_REQ, TESTDEF_VM,
TESTDEF_DEFINE, TESTDEF_DEFINE_TEST, TESTDEF_DEBUG_UNIT_SUPPRESS, TESTDEF_DB, TESTDEF_BIN_REQ, TESTDEF_VM,
TESTDEF_CONTAINER_REQUIRED)
{
$hTestDefHash->{$strModule}{$strTest}{$strVar} = coalesce(

14
test/lib/pgBackRestTest/Common/JobTest.pm
View File

@ -233,8 +233,8 @@ sub run
$rhBuildInit->{$self->{oTest}->{&TEST_VM}}{$self->{iVmIdx}} = true;
}
# If testing Perl code (or C code that calls Perl code) install bin and Perl C Library
if ($self->{oTest}->{&TEST_VM} ne VM_NONE && (!$self->{oTest}->{&TEST_C} || $self->{oTest}->{&TEST_PERL_REQ}))
# If testing Perl code (or C code that calls the binary) install binary
if ($self->{oTest}->{&TEST_VM} ne VM_NONE && (!$self->{oTest}->{&TEST_C} || $self->{oTest}->{&TEST_BIN_REQ}))
{
jobInstallC($self->{strBackRestBase}, $self->{oTest}->{&TEST_VM}, $strImage);
}
@ -433,8 +433,7 @@ sub run
buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/test.c", $strTestC);
# Create build.auto.h
my $strBuildAutoH =
"#define HAVE_LIBPERL\n";
my $strBuildAutoH = "";
buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/" . BUILD_AUTO_H, $strBuildAutoH);
@ -451,8 +450,7 @@ sub run
# Flags that are common to all builds
my $strCommonFlags =
'-I. -Itest -std=c99 -fPIC -g -Wno-clobbered -D_POSIX_C_SOURCE=200112L' .
' `perl -MExtUtils::Embed -e ccopts`' .
'-I. -Itest -std=c99 -fPIC -g -Wno-clobbered -D_POSIX_C_SOURCE=200809L -D_FILE_OFFSET_BITS=64' .
' `xml2-config --cflags`' . ($self->{bProfile} ? " -pg" : '') .
' -I`pg_config --includedir`' .
($self->{oTest}->{&TEST_DEBUG_UNIT_SUPPRESS} ? '' : " -DDEBUG_UNIT") .
@ -504,7 +502,6 @@ sub run
"LDFLAGS=-lcrypto -lssl -lxml2 -lz" .
(vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .
(vmWithBackTrace($self->{oTest}->{&TEST_VM}) && $self->{bBackTrace} ? ' -lbacktrace' : '') .
" `perl -MExtUtils::Embed -e ldopts`\n" .
"\n" .
"SRCS=" . join(' ', @stryCFile) . "\n" .
"OBJS=\$(SRCS:.c=.o)\n" .
@ -783,7 +780,7 @@ sub end
}
####################################################################################################################################
# Install C binary and library
# Install C binary
####################################################################################################################################
sub jobInstallC
{
@ -791,7 +788,6 @@ sub jobInstallC
my $strVm = shift;
my $strImage = shift;
# Install Perl C Library
my $oVm = vmGet();
my $strBuildPath = "${strBasePath}/test/.vagrant/bin/${strVm}";
my $strBuildLibCPath = "${strBuildPath}/libc";

6
test/lib/pgBackRestTest/Common/ListTest.pm
View File

@ -38,8 +38,8 @@ use constant TEST_MODULE => 'module';
push @EXPORT, qw(TEST_MODULE);
use constant TEST_NAME => 'test';
push @EXPORT, qw(TEST_NAME);
use constant TEST_PERL_REQ => 'perl-req';
push @EXPORT, qw(TEST_PERL_REQ);
use constant TEST_BIN_REQ => 'bin-req';
push @EXPORT, qw(TEST_BIN_REQ);
use constant TEST_PGSQL_BIN => 'pgsql-bin';
push @EXPORT, qw(TEST_PGSQL_BIN);
use constant TEST_INTEGRATION => 'integration';
@ -172,7 +172,7 @@ sub testListGet
&TEST_CONTAINER => defined($hTest->{&TESTDEF_CONTAINER}) ?
$hTest->{&TESTDEF_CONTAINER} : $hModule->{&TESTDEF_CONTAINER},
&TEST_PGSQL_BIN => $strPgSqlBin,
&TEST_PERL_REQ => $hTest->{&TESTDEF_PERL_REQ},
&TEST_BIN_REQ => $hTest->{&TESTDEF_BIN_REQ},
&TEST_INTEGRATION => $hTest->{&TESTDEF_INTEGRATION},
&TEST_MODULE => $strModule,
&TEST_NAME => $strModuleTest,

62
test/lib/pgBackRestTest/Module/Common/CommonEncodePerlTest.pm
View File

@ -1,62 +0,0 @@
####################################################################################################################################
# Test Binary to String Encode/Decode
####################################################################################################################################
package pgBackRestTest::Module::Common::CommonEncodePerlTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::LibC qw(:encode);
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
################################################################################################################################
if ($self->begin("encodeToStrBase64() and decodeToBinBase64()"))
{
my $strData = 'string_to_encode';
my $strEncodedData = 'c3RyaW5nX3RvX2VuY29kZQ==';
$self->testResult(sub {encodeToStr(ENCODE_TYPE_BASE64, $strData)}, $strEncodedData, 'encode string');
$self->testResult(sub {decodeToBin(ENCODE_TYPE_BASE64, $strEncodedData)}, $strData, 'decode string');
#---------------------------------------------------------------------------------------------------------------------------
my $tData =
pack('C', 1) . pack('C', 2) . pack('C', 3) . pack('C', 4) . pack('C', 5) . pack('C', 4) . pack('C', 3) . pack('C', 2);
my $tEncodedData = 'AQIDBAUEAwI=';
$self->testResult(sub {encodeToStr(ENCODE_TYPE_BASE64, $tData)}, $tEncodedData, 'encode binary');
$self->testResult(sub {decodeToBin(ENCODE_TYPE_BASE64, $tEncodedData)}, $tData, 'decode binary');
#---------------------------------------------------------------------------------------------------------------------------
$tData .= pack('C', 1);
$tEncodedData = 'AQIDBAUEAwIB';
$self->testResult(sub {encodeToStr(ENCODE_TYPE_BASE64, $tData)}, $tEncodedData, 'encode binary');
$self->testResult(sub {decodeToBin(ENCODE_TYPE_BASE64, $tEncodedData)}, $tData, 'decode binary');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {encodeToStr(99999, '')}, ERROR_ASSERT, 'invalid encode type 99999');
$self->testException(
sub {decodeToBin(ENCODE_TYPE_BASE64, "XX")}, ERROR_FORMAT, 'base64 size 2 is not evenly divisible by 4');
}
}
1;

533
test/lib/pgBackRestTest/Module/Common/CommonIniPerlTest.pm
View File

@ -1,533 +0,0 @@
####################################################################################################################################
# Unit tests for Ini module
####################################################################################################################################
package pgBackRestTest::Module::Common::CommonIniPerlTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Storage::Base;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# iniHeader
####################################################################################################################################
sub iniHeader
{
my $self = shift;
my $oIni = shift;
my $iFormat = shift;
my $iVersion = shift;
my $strChecksum = shift;
return
"[backrest]\n" .
"backrest-format=" . (defined($iFormat) ? $iFormat : $oIni->get(INI_SECTION_BACKREST, INI_KEY_FORMAT)) . "\n" .
"backrest-version=\"" . (defined($iVersion) ? $iVersion : $oIni->get(INI_SECTION_BACKREST, INI_KEY_VERSION)) . "\"\n" .
"\n" .
"[backrest]\n" .
"backrest-checksum=\"" .
(defined($strChecksum) ? $strChecksum : $oIni->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM)) . "\"\n";
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test ini file
my $strTestFile = $self->testPath() . '/test.ini';
my $strTestFileCopy = "${strTestFile}.copy";
# Test keys, values
my $strSection = 'test-section';
my $strKey = 'test-key';
my $strSubKey = 'test-subkey';
my $strValue = 'test-value';
################################################################################################################################
if ($self->begin("iniRender()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIni = {section1 => {key => 'value'}, section2 => {key => ['value1', 'value2']}};
$self->testResult(
sub {iniRender($oIni, true)}, "[section1]\nkey=value\n\n[section2]\nkey=value1\nkey=value2\n",
'relaxed formatting');
$self->testResult(
sub {iniRender($oIni, false)}, "[section1]\nkey=\"value\"\n\n[section2]\nkey=[\"value1\",\"value2\"]\n",
'strict formatting');
}
################################################################################################################################
if ($self->begin("iniParse()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strIni = "[section]\nkey";
$self->testException(sub {iniParse($strIni)}, ERROR_CONFIG, "unable to find '=' in 'key'");
#---------------------------------------------------------------------------------------------------------------------------
$strIni = "[section]\n# comment\n\nkey=\"value\"";
$self->testResult(sub {iniParse($strIni)}, '{section => {key => value}}', 'ignore blank lines and comments');
#---------------------------------------------------------------------------------------------------------------------------
$strIni = "[section]\nkey=value";
$self->testResult(sub {iniParse($strIni, {bRelaxed => true})}, '{section => {key => value}}', 'relaxed value read');
#---------------------------------------------------------------------------------------------------------------------------
$strIni = "[section]\nkey=value1\nkey=value2\nkey=value3";
$self->testResult(
sub {iniParse($strIni, {bRelaxed => true})}, '{section => {key => (value1, value2, value3)}}', 'relaxed array read');
}
################################################################################################################################
if ($self->begin("Ini->new()"))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {(new pgBackRest::Common::Ini($strTestFile, {bIgnoreMissing => true}))->exists()}, false, 'ignore missing');
#---------------------------------------------------------------------------------------------------------------------------
my $oIni = new pgBackRest::Common::Ini(
$strTestFile, {bLoad => false, iInitFormat => 4, strInitVersion => '1.01'});
$self->testResult($oIni->exists(), false, 'file does not exist');
$oIni->saveCopy();
$self->testResult($oIni->exists(), false, 'file does not exist after saveCopy()');
$self->testResult(
sub {${storageTest()->get($strTestFile . INI_COPY_EXT)}},
$self->iniHeader(undef, 4, '1.01', '488e5ca1a018cd7cd6d4e15150548f39f493dacd'),
'empty with synthetic format and version');
#---------------------------------------------------------------------------------------------------------------------------
$oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
$oIni->saveCopy();
$self->testResult(
sub {${storageTest()->get($strTestFile . INI_COPY_EXT)}},
$self->iniHeader(undef, REPOSITORY_FORMAT, PROJECT_VERSION, $oIni->hash()),
'empty with default format and version');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageTest()->list($self->testPath())},
'test.ini.copy',
'only copy is saved');
$oIni->save();
$self->testResult(
sub {storageTest()->list($self->testPath())},
'(test.ini, test.ini.copy)',
'both versions are saved');
$self->testException(
sub {$oIni->saveCopy()}, ERROR_ASSERT,
"cannot save copy only when '${strTestFile}' exists");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'normal load');
#---------------------------------------------------------------------------------------------------------------------------
my $hIni = iniParse(${storageTest()->get($strTestFile)});
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = BOGUS;
storageTest()->put($strTestFile, iniRender($hIni));
storageTest()->put($strTestFileCopy, iniRender($hIni));
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "invalid checksum in '${strTestFile}', expected '" .
# $oIni->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM) . "' but found 'bogus'");
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = $oIni->hash();
storageTest()->put($strTestFile, iniRender($hIni));
#---------------------------------------------------------------------------------------------------------------------------
$oIni->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, REPOSITORY_FORMAT - 1);
$oIni->save();
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "invalid format in '${strTestFile}', expected " . REPOSITORY_FORMAT . ' but found ' . (REPOSITORY_FORMAT - 1));
$oIni->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, REPOSITORY_FORMAT);
$oIni->save();
#---------------------------------------------------------------------------------------------------------------------------
$oIni->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, '1.01');
$oIni->save();
$self->testResult(
sub {${storageTest()->get($strTestFile . INI_COPY_EXT)}},
$self->iniHeader($oIni, undef, '1.01'),
'verify old version was written');
$oIni = new pgBackRest::Common::Ini($strTestFile);
$self->testResult(sub {$oIni->get(INI_SECTION_BACKREST, INI_KEY_VERSION)}, PROJECT_VERSION, 'version is updated on load');
$oIni->save();
$self->testResult(
sub {${storageTest()->get($strTestFile . INI_COPY_EXT)}},
$self->iniHeader($oIni, undef, PROJECT_VERSION),
'verify version is updated on load');
$self->testResult(sub {$oIni->save()}, false, 'save again with no changes');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {new pgBackRest::Common::Ini($strTestFile, {bLoad => false, strContent => ${storageTest()->get($strTestFile)}})},
'[object]', 'new() passing content as a string');
#---------------------------------------------------------------------------------------------------------------------------
executeTest("rm -rf ${strTestFile}*");
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strTestFileCopy, BOGUS);
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "key/value pair 'bogus' found outside of a section");
#---------------------------------------------------------------------------------------------------------------------------
my $oIniSource = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
storageTest()->put($strTestFileCopy, iniRender($oIniSource->{oContent}));
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "invalid checksum in '${strTestFileCopy}', expected '" .
# $oIniSource->hash() . "' but found [undef]");
#---------------------------------------------------------------------------------------------------------------------------
$oIniSource = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
$oIniSource->save();
storageTest()->put($strTestFile, BOGUS);
# main invalid, copy ok
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'invalid main - load copy');
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->remove($strTestFileCopy);
storageTest()->put($strTestFile, "[section]\n" . BOGUS);
# main invalid, copy missing
$self->testException
(sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
#"unable to find '=' in 'bogus'");
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strTestFile, iniRender($oIniSource->{oContent}));
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'main ok, copy missing');
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strTestFileCopy, BOGUS);
# main ok, copy invalid
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'main ok, copy invalid');
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strTestFile, BOGUS);
# main invalid, copy invalid
$self->testException(
sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_FILE_MISSING,
"unable to open ${strTestFile} or ${strTestFile}" . INI_COPY_EXT);
# "key/value pair 'bogus' found outside of a section");
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put($strTestFile, iniRender($hIni));
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = BOGUS;
storageTest()->put($strTestFileCopy, iniRender($hIni));
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'invalid copy header - load main');
#---------------------------------------------------------------------------------------------------------------------------
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = $oIni->hash();
storageTest()->put($strTestFileCopy, iniRender($hIni));
$hIni->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = BOGUS;
storageTest()->put($strTestFile, iniRender($hIni));
$self->testResult(sub {new pgBackRest::Common::Ini($strTestFile)}, '[object]', 'invalid main header - load copy');
#---------------------------------------------------------------------------------------------------------------------------
# Prepend encryption Magic signature to copy (main invalid) to simulate encryption
executeTest('echo "' . CIPHER_MAGIC . '$(cat ' . $strTestFileCopy . ')" > ' . $strTestFileCopy);
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_CRYPTO,
"unable to parse '$strTestFileCopy'" .
"\nHINT: is or was the repo encrypted?");
# Prepend encryption Magic signature to main to simulate encryption
executeTest('echo "' . CIPHER_MAGIC . '$(cat ' . $strTestFile . ')" > ' . $strTestFile);
$self->testException(sub {new pgBackRest::Common::Ini($strTestFile)}, ERROR_CRYPTO,
"unable to parse '$strTestFile'" .
"\nHINT: is or was the repo encrypted?");
}
################################################################################################################################
if ($self->begin("Ini->set()"))
{
my $oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oIni->set()}, ERROR_ASSERT, 'strSection and strKey are required');
$self->testException(sub {$oIni->set($strSection)}, ERROR_ASSERT, 'strSection and strKey are required');
$self->testException(sub {$oIni->set(undef, $strKey)}, ERROR_ASSERT, 'strSection and strKey are required');
#---------------------------------------------------------------------------------------------------------------------------
$oIni->{bModified} = false;
$self->testResult(sub {$oIni->set($strSection, $strKey, undef, $strValue)}, true, 'set key value');
$self->testResult($oIni->modified(), true, ' check changed flag = true');
#---------------------------------------------------------------------------------------------------------------------------
$oIni->{bModified} = false;
$self->testResult(sub {$oIni->set($strSection, $strKey, undef, $strValue)}, false, 'set same key value');
$self->testResult($oIni->modified(), false, ' check changed flag remains false');
#---------------------------------------------------------------------------------------------------------------------------
$oIni->{bModified} = false;
$self->testResult(sub {$oIni->set($strSection, $strKey, undef, "${strValue}2")}, true, 'set different key value');
$self->testResult($oIni->modified(), true, ' check changed flag = true');
$self->testResult(sub {$oIni->get($strSection, $strKey)}, "${strValue}2", 'get last key value');
#---------------------------------------------------------------------------------------------------------------------------
$oIni->{bModified} = false;
$self->testResult(sub {$oIni->set($strSection, $strKey, undef, undef)}, true, 'set undef key value');
$self->testResult($oIni->modified(), true, ' check changed flag = true');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->set($strSection, "${strKey}2", $strSubKey, $strValue)}, true, 'set subkey value');
}
################################################################################################################################
if ($self->begin("Ini->get()"))
{
my $oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oIni->get()}, ERROR_ASSERT, 'strSection is required');
$self->testException(sub {$oIni->get($strSection)}, ERROR_ASSERT, "strSection '${strSection}' is required but not defined");
$self->testException(
sub {$oIni->get($strSection, $strKey, undef)}, ERROR_ASSERT,
"strSection '${strSection}', strKey '${strKey}' is required but not defined");
$self->testException(
sub {$oIni->get($strSection, undef, $strSubKey)}, ERROR_ASSERT,
"strKey is required when strSubKey '${strSubKey}' is requested");
$self->testException(
sub {$oIni->get($strSection, $strKey, $strSubKey, true)}, ERROR_ASSERT,
"strSection '${strSection}', strKey '${strKey}', strSubKey '${strSubKey}' is required but not defined");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->get($strSection, undef, undef, false)}, '[undef]', 'section value is not required');
$self->testResult(sub {$oIni->get($strSection, undef, undef, false, $strValue)}, $strValue, 'section value is defaulted');
#---------------------------------------------------------------------------------------------------------------------------
$oIni->set($strSection, $strKey, $strSubKey, $strValue);
$self->testResult(sub {$oIni->get($strSection, "${strKey}2", "${strSubKey}2", false)}, undef, 'missing key value');
$self->testResult(sub {$oIni->get($strSection, $strKey, "${strSubKey}2", false)}, undef, 'missing subkey value');
$self->testResult(sub {$oIni->get($strSection, $strKey, $strSubKey)}, $strValue, 'get subkey value');
$self->testResult(sub {$oIni->get($strSection, $strKey)}, "{${strSubKey} => ${strValue}}", 'get key value');
$self->testResult(sub {$oIni->get($strSection)}, "{${strKey} => {${strSubKey} => ${strValue}}}", 'get section value');
}
################################################################################################################################
if ($self->begin("Ini->remove()"))
{
my $oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oIni->remove()}, ERROR_ASSERT, 'strSection is required');
$self->testException(
sub {$oIni->remove($strSection, undef, $strSubKey)}, ERROR_ASSERT,
"strKey is required when strSubKey '${strSubKey}' is requested");
#---------------------------------------------------------------------------------------------------------------------------
$oIni->{bModified} = false;
$self->testResult(sub {$oIni->remove($strSection)}, false, 'undefined section is not removed');
$self->testResult($oIni->modified(), '0', ' check changed flag remains false');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->set($strSection, $strKey, undef, $strValue)}, true, 'set key');
$oIni->{bModified} = false;
$self->testResult(sub {$oIni->remove($strSection, $strKey)}, true, ' remove key');
$self->testResult($oIni->modified(), '1', ' check changed flag = true');
$self->testResult(sub {$oIni->test($strSection, $strKey)}, false, ' test key');
$self->testResult(sub {$oIni->test($strSection)}, false, ' test section');
$self->testResult(sub {$oIni->set($strSection, $strKey, undef, $strValue)}, true, 'set key 1');
$self->testResult(sub {$oIni->set($strSection, "${strKey}2", undef, $strValue)}, true, ' set key 2');
$self->testResult(sub {$oIni->remove($strSection, $strKey)}, true, ' remove key 1');
$self->testResult(sub {$oIni->test($strSection, $strKey)}, false, ' test key');
$self->testResult(sub {$oIni->test($strSection)}, true, ' test section');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->set($strSection, $strKey, undef, $strValue)}, true, 'set key');
$self->testResult(sub {$oIni->remove($strSection)}, true, ' remove section');
$self->testResult(sub {$oIni->test($strSection)}, false, ' test section');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->set($strSection, $strKey, $strSubKey, $strValue)}, true, 'set subkey');
$self->testResult(sub {$oIni->remove($strSection, $strKey, $strSubKey)}, true, ' remove subkey');
$self->testResult(sub {$oIni->test($strSection, $strKey, $strSubKey)}, false, ' test subkey');
$self->testResult(sub {$oIni->test($strSection, $strKey)}, true, ' test key');
$self->testResult(sub {$oIni->test($strSection)}, true, ' test section');
$self->testResult(sub {$oIni->set($strSection, $strKey, $strSubKey, $strValue)}, true, 'set subkey 1');
$self->testResult(sub {$oIni->set($strSection, $strKey, "${strSubKey}2", $strValue)}, true, 'set subkey 2');
$self->testResult(sub {$oIni->remove($strSection, $strKey, $strSubKey)}, true, ' remove subkey');
$self->testResult(sub {$oIni->test($strSection, $strKey, $strSubKey)}, false, ' test subkey');
$self->testResult(sub {$oIni->test($strSection, $strKey)}, true, ' test key');
$self->testResult(sub {$oIni->test($strSection)}, true, ' test section');
}
################################################################################################################################
if ($self->begin("Ini->test()"))
{
my $oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->test($strSection, $strKey)}, false, 'test undefined key');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->set($strSection, $strKey, undef, $strValue)}, true, 'define key');
$self->testResult(sub {$oIni->test($strSection, $strKey)}, true, 'test key exists');
$self->testResult(sub {$oIni->test($strSection, $strKey, undef, $strValue)}, true, 'test key value');
$self->testResult(sub {$oIni->test($strSection, $strKey, undef, BOGUS)}, false, 'test key invalid value');
}
################################################################################################################################
if ($self->begin("Ini->boolSet() & Ini->boolGet() & Ini->boolTest()"))
{
my $oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->boolTest($strSection, $strKey)}, false, 'test bool on undefined key');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->boolGet($strSection, $strKey, undef, false, false)}, false, 'get bool default false value');
$self->testResult(sub {$oIni->boolGet($strSection, $strKey, undef, false, true)}, true, 'get bool default true value');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->boolSet($strSection, $strKey, undef, undef)}, true, 'set bool false (undef) value');
$self->testResult(sub {$oIni->boolGet($strSection, $strKey)}, false, ' check bool false value');
$self->testResult(sub {$oIni->boolTest($strSection, $strKey, undef, false)}, true, 'test bool on false key');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->boolSet($strSection, $strKey, undef, false)}, false, 'set bool false value');
$self->testResult(sub {$oIni->boolGet($strSection, $strKey)}, false, ' check value');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->boolSet($strSection, $strKey, undef, true)}, true, 'set bool false value');
$self->testResult(sub {$oIni->boolGet($strSection, $strKey)}, true, ' check value');
$self->testResult(sub {$oIni->boolTest($strSection, $strKey, undef, true)}, true, 'test bool on true key');
}
################################################################################################################################
if ($self->begin("Ini->numericSet() & Ini->numericGet() & Ini->numericTest()"))
{
my $oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->numericSet($strSection, $strKey)}, true, 'set numeric undef value');
$self->testResult(sub {$oIni->test($strSection, $strKey)}, false, 'test numeric undef value');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->numericGet($strSection, $strKey, undef, false, 1000)}, 1000, 'get numeric default value');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->numericSet($strSection, $strKey, undef, 0)}, true, 'set numeric 0 value');
$self->testResult(sub {$oIni->numericGet($strSection, $strKey)}, 0, ' check value');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->numericSet($strSection, $strKey, undef, 0)}, false, 'set numeric 0 value again');
$self->testResult(sub {$oIni->numericGet($strSection, $strKey)}, 0, ' check value');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->numericSet($strSection, $strKey, undef, -100)}, true, 'set numeric -100 value');
$self->testResult(sub {$oIni->numericGet($strSection, $strKey)}, -100, ' check value');
}
################################################################################################################################
if ($self->begin("Ini->keys()"))
{
my $oIni = new pgBackRest::Common::Ini($strTestFile, {bLoad => false});
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->keys($strSection)}, '[undef]', 'section undefined');
#---------------------------------------------------------------------------------------------------------------------------
$oIni->set($strSection, 'a', undef, $strValue);
$oIni->set($strSection, 'c', undef, $strValue);
$oIni->set($strSection, 'b', undef, $strValue);
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIni->keys($strSection)}, '(a, b, c)', 'sort forward (default)');
$self->testResult(sub {$oIni->keys($strSection, INI_SORT_FORWARD)}, '(a, b, c)', 'sort forward');
$self->testResult(sub {$oIni->keys($strSection, INI_SORT_REVERSE)}, '(c, b, a)', 'sort reverse');
$self->testResult(sub {sort($oIni->keys($strSection, INI_SORT_NONE))}, '(a, b, c)', 'sort none');
$self->testException(sub {sort($oIni->keys($strSection, BOGUS))}, ERROR_ASSERT, "invalid strSortOrder '" . BOGUS . "'");
}
}
1;

259
test/lib/pgBackRestTest/Module/Common/CommonIoBufferedPerlTest.pm
View File

@ -1,259 +0,0 @@
####################################################################################################################################
# Tests for Common::Io::Buffered module
####################################################################################################################################
package pgBackRestTest::Module::Common::CommonIoBufferedPerlTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use IO::Socket::UNIX;
use Time::HiRes qw(usleep);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# socketServer
####################################################################################################################################
sub socketServer
{
my $self = shift;
my $strSocketFile = shift;
my $fnServer = shift;
# Fork off the server
if (fork() == 0)
{
# Open the domain socket
my $oSocketServer = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Local => $strSocketFile, Listen => 1);
&log(INFO, " * socket server open");
# Wait for a connection and create IO object
my $oConnection = $oSocketServer->accept();
my $oIoHandle = new pgBackRest::Common::Io::Handle('socket server', $oConnection, $oConnection);
&log(INFO, " * socket server connected");
# Run server function
$fnServer->($oIoHandle);
# Shutdown server
$oSocketServer->close();
&log(INFO, " * socket server closed");
unlink($strSocketFile);
exit 0;
}
# Wait for client socket
my $oWait = waitInit(5);
while (!-e $strSocketFile && waitMore($oWait)) {};
# Open the client socket
my $oClient = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Peer => $strSocketFile);
if (!defined($oClient))
{
logErrorResult(ERROR_FILE_OPEN, 'unable to open client socket', $OS_ERROR);
}
&log(INFO, " * socket client connected");
return $oClient;
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strSocketFile = $self->testPath() . qw{/} . 'domain.socket';
################################################################################################################################
if ($self->begin('new() & timeout() & bufferMax()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoBuffered = $self->testResult(
sub {new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('test'), 10, 2048)}, '[object]', 'new - no handles');
$self->testResult(sub {$oIoBuffered->timeout()}, 10, ' check timeout');
$self->testResult(sub {$oIoBuffered->bufferMax()}, 2048, ' check buffer max');
}
################################################################################################################################
if ($self->begin('readLine() & writeLine()'))
{
my $oClient = $self->socketServer($strSocketFile, sub
{
my $oIoHandle = shift;
my $tResponse = '';
my $tData;
# Ack after timeout
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write 8 char line
$tResponse = '';
$tData = "12345678\n";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write 3 lines
$tResponse = '';
$tData = "1\n2\n345678\n";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write blank line
$tResponse = '';
$tData = "\n";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write char with no linefeed
$tResponse = '';
$tData = "A";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write linefeed
$tResponse = '';
$tData = "\n";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
});
#---------------------------------------------------------------------------------------------------------------------------
my $oIoBuffered = $self->testResult(
sub {new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('socket client', $oClient, $oClient), 1, 4)}, '[object]', 'open');
$self->testException(
sub {$oIoBuffered->readLine()}, ERROR_FILE_READ, 'unable to read line after 1 second(s) from socket client');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine()}, '12345678', 'read 8 char line');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine()}, '1', 'read 1 char line');
$self->testResult(sub {$oIoBuffered->readLine()}, '2', 'read 1 char line');
$self->testResult(sub {$oIoBuffered->readLine()}, '345678', 'read 6 char line');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine()}, '', 'read blank line');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine(undef, false)}, undef, 'read ignoring error');
$self->testException(
sub {$oIoBuffered->readLine(undef, true)}, ERROR_FILE_READ,
'unable to read line after 1 second(s) from socket client');
# Clear buffer so EOF tests pass
$oIoBuffered->writeLine();
$oIoBuffered->readLine();
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oIoBuffered->readLine()}, ERROR_FILE_READ, 'unexpected EOF reading line from socket client');
$self->testException(
sub {$oIoBuffered->readLine(false)}, ERROR_FILE_READ, 'unexpected EOF reading line from socket client');
$self->testResult(sub {$oIoBuffered->readLine(true)}, undef, 'ignore EOF');
}
################################################################################################################################
if ($self->begin('read'))
{
my $tBuffer;
my $oClient = $self->socketServer($strSocketFile, sub
{
my $oIoHandle = shift;
my $tResponse = '';
my $tData;
# Ack after timeout
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write mixed buffer
$tResponse = '';
$tData = "123\n123\n1";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
# Write buffer
$tResponse = '';
$tData = "23456789";
$oIoHandle->write(\$tData);
# Wait before writing the rest to force client to loop
usleep(500);
$tData = "0";
$oIoHandle->write(\$tData);
while (length($tResponse) != 1) {$oIoHandle->read(\$tResponse, 1)};
});
#---------------------------------------------------------------------------------------------------------------------------
my $oIoBuffered = $self->testResult(
sub {new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('socket client', $oClient, $oClient), 1, 8)}, '[object]', 'open');
$self->testException(
sub {$oIoBuffered->read(\$tBuffer, 1, true)}, ERROR_FILE_READ,
'unable to read 1 byte(s) after 1 second(s) from socket client');
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 1)}, 0, ' read 0 char buffer');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->readLine()}, '123', ' read 3 char line');
undef($tBuffer);
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 1)}, 1, ' read 1 char buffer');
$self->testResult(sub {$tBuffer}, '1', ' check 1 char buffer');
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 3)}, 3, ' read 3 char buffer');
$self->testResult(sub {$tBuffer}, "123\n", ' check 3 char buffer');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
undef($tBuffer);
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 10)}, 10, ' read 10 char buffer');
$self->testResult(sub {$tBuffer}, '1234567890', ' check 10 char buffer');
$oIoBuffered->writeLine();
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoBuffered->read(\$tBuffer, 5)}, 0, ' expect EOF');
$self->testException(
sub {$oIoBuffered->read(\$tBuffer, 5, true)}, ERROR_FILE_READ,
'unable to read 5 byte(s) due to EOF from socket client');
}
}
1;

209
test/lib/pgBackRestTest/Module/Common/CommonIoHandlePerlTest.pm
View File

@ -1,209 +0,0 @@
####################################################################################################################################
# Tests for Common::Io::Handle module
####################################################################################################################################
package pgBackRestTest::Module::Common::CommonIoHandlePerlTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC O_NONBLOCK);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Base;
use pgBackRest::Common::Io::Handle;
use pgBackRest::Common::Log;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strFile = $self->testPath() . qw{/} . 'file.txt';
my $strFileContent = 'TESTDATA';
my $iFileLength = length($strFileContent);
my $iFileLengthHalf = int($iFileLength / 2);
################################################################################################################################
if ($self->begin('new() & handleRead() & handleReadSet() & handleWrite() & handleWriteSet()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, undef)}, '[object]', 'new - no handles');
$self->testResult(sub {$oIoHandle->id()}, 'test', ' check error msg');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oIoHandle->handleReadSet(1)}, 1, ' set read handle');
$self->testResult(sub {$oIoHandle->handleRead()}, 1, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, undef, ' check write handle');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oIoHandle->handleWriteSet(2)}, 2, ' set write handle');
$self->testResult(sub {$oIoHandle->handleRead()}, 1, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, 2, ' check write handle');
#---------------------------------------------------------------------------------------------------------------------------
$oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', 1, undef)}, '[object]', 'new - read handle');
$self->testResult(sub {$oIoHandle->handleRead()}, 1, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, undef, ' check write handle');
#---------------------------------------------------------------------------------------------------------------------------
$oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, 2)}, '[object]', 'new - write handle');
$self->testResult(sub {$oIoHandle->handleRead()}, undef, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, 2, ' check write handle');
#---------------------------------------------------------------------------------------------------------------------------
$oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', 1, 2)}, '[object]', 'new - read/write handle');
$self->testResult(sub {$oIoHandle->handleRead()}, 1, ' check read handle');
$self->testResult(sub {$oIoHandle->handleWrite()}, 2, ' check write handle');
}
################################################################################################################################
if ($self->begin('read()'))
{
my $tContent;
my $fhRead;
#---------------------------------------------------------------------------------------------------------------------------
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
sysopen($fhRead, $strFile, O_RDONLY)
or confess &log(ERROR, "unable to open '${strFile}'");
my $oIoHandle = $self->testResult(sub {new pgBackRest::Common::Io::Handle("'$strFile'", $fhRead)}, '[object]', 'open');
$self->testException(
sub {$oIoHandle->read(\$tContent, -1)}, ERROR_FILE_READ, "unable to read from '${strFile}': Negative length");
#---------------------------------------------------------------------------------------------------------------------------
$tContent = undef;
sysopen($fhRead, $strFile, O_RDONLY)
or confess &log(ERROR, "unable to open '${strFile}'");
$oIoHandle = $self->testResult(sub {new pgBackRest::Common::Io::Handle("'$strFile'", $fhRead)}, '[object]', 'open');
$self->testResult(sub {$oIoHandle->read(\$tContent, $iFileLengthHalf)}, $iFileLengthHalf, ' read part 1');
$self->testResult($tContent, substr($strFileContent, 0, $iFileLengthHalf), ' check read');
$self->testResult(sub {$oIoHandle->eof()}, false, ' not eof');
$self->testResult(
sub {$oIoHandle->read(
\$tContent, $iFileLength - $iFileLengthHalf)}, $iFileLength - $iFileLengthHalf, ' read part 2');
$self->testResult($tContent, $strFileContent, ' check read');
$self->testResult(sub {$oIoHandle->read(\$tContent, 1)}, 0, ' zero read');
$self->testResult(sub {$oIoHandle->eof()}, true, ' eof');
}
################################################################################################################################
if ($self->begin('write()'))
{
my $tContent;
my $fhRead;
my $fhWrite;
#---------------------------------------------------------------------------------------------------------------------------
sysopen($fhWrite, $strFile, O_WRONLY | O_CREAT | O_TRUNC)
or confess &log(ERROR, "unable to open '${strFile}'");
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle("'$strFile'", undef, $fhWrite)}, '[object]', 'open write');
$self->testException(
sub {$oIoHandle->write(undef)}, ERROR_FILE_WRITE,
"unable to write to '${strFile}': Can't use an undefined value as a SCALAR reference");
#---------------------------------------------------------------------------------------------------------------------------
$tContent = substr($strFileContent, 0, $iFileLengthHalf);
$self->testResult(sub {$oIoHandle->write(\$tContent)}, $iFileLengthHalf, ' write part 1');
$self->testResult(sub {$oIoHandle->size()}, $iFileLengthHalf, ' check part 1 size');
$tContent = substr($strFileContent, $iFileLengthHalf);
$self->testResult(sub {$oIoHandle->write(\$tContent)}, $iFileLength - $iFileLengthHalf, ' write part 2');
$self->testResult(sub {$oIoHandle->size()}, $iFileLength, ' check part 2 size');
$self->testResult(sub {$oIoHandle->close()}, true, ' close');
sysopen($fhRead, $strFile, O_RDONLY)
or confess &log(ERROR, "unable to open '${strFile}'");
$oIoHandle = $self->testResult(sub {new pgBackRest::Common::Io::Handle("'$strFile'", $fhRead)}, '[object]', 'open read');
$tContent = undef;
$self->testResult(sub {$oIoHandle->read(\$tContent, $iFileLength)}, $iFileLength, ' read');
$self->testResult($tContent, $strFileContent, ' check write content');
}
################################################################################################################################
if ($self->begin('result() & resultSet()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, undef)}, '[object]', 'new - no handles');
$self->testResult(sub {$oIoHandle->resultSet('Module::1', 1)}, 1, ' set int result');
$self->testResult(sub {$oIoHandle->resultSet('Module::2', {value => 2})}, '{value => 2}', ' set hash result');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoHandle->result('Module::1')}, 1, ' check int result');
$self->testResult(sub {$oIoHandle->result('Module::2')}, '{value => 2}', ' check hash result');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oIoHandle->{rhResult}}, '{Module::1 => 1, Module::2 => {value => 2}}', ' check all results');
}
################################################################################################################################
if ($self->begin('className()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoHandle = $self->testResult(
sub {new pgBackRest::Common::Io::Handle('test', undef, undef)}, '[object]', 'new - no handles');
$self->testResult(sub {$oIoHandle->className()}, COMMON_IO_HANDLE, ' check class name');
}
################################################################################################################################
if ($self->begin('close()'))
{
my $tContent;
my $fhRead;
#---------------------------------------------------------------------------------------------------------------------------
executeTest("echo -n '${strFileContent}' | tee ${strFile}");
sysopen($fhRead, $strFile, O_RDONLY)
or confess &log(ERROR, "unable to open '${strFile}'");
my $oIoHandle = $self->testResult(sub {new pgBackRest::Common::Io::Handle("'$strFile'", $fhRead)}, '[object]', 'open read');
$self->testResult(sub {$oIoHandle->read(\$tContent, $iFileLengthHalf)}, $iFileLengthHalf, ' read');
$self->testResult(sub {$oIoHandle->close()}, true, ' close');
$self->testResult(sub {$oIoHandle->close()}, true, ' close again');
$self->testResult(sub {$oIoHandle->result(COMMON_IO_HANDLE)}, $iFileLengthHalf, ' check result');
# Destroy and to make sure close runs again
undef($oIoHandle);
}
}
1;

72
test/lib/pgBackRestTest/Module/Common/CommonIoProcessPerlTest.pm
View File

@ -1,72 +0,0 @@
####################################################################################################################################
# Tests for Common::Io::Process module
####################################################################################################################################
package pgBackRestTest::Module::Common::CommonIoProcessPerlTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Io::Process;
use pgBackRest::Common::Log;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strFile = $self->testPath() . qw{/} . 'file.txt';
my $strFileContent = 'TESTDATA';
################################################################################################################################
if ($self->begin('new() & processId()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoProcess = $self->testResult(sub {
new pgBackRest::Common::Io::Process(
new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('test'), 1, 32), "echo '${strFileContent}'")}, '[object]', 'new - echo');
$self->testResult(sub {defined($oIoProcess->processId())}, true, ' process id defined');
}
################################################################################################################################
if ($self->begin('close() and error when stderr has data'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoProcess =
new pgBackRest::Common::Io::Process(
new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('test'), 1, 32), "echo '${strFileContent}' 1>&2");
$self->testException(
sub {$oIoProcess->close()}, ERROR_FILE_READ, 'test terminated unexpectedly [000]: TESTDATA');
}
################################################################################################################################
if ($self->begin('close() & error()'))
{
#---------------------------------------------------------------------------------------------------------------------------
my $oIoProcess =
new pgBackRest::Common::Io::Process(
new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('test'), 1, 32), "echo '${strFileContent}'");
$oIoProcess->close();
$self->testException(
sub {$oIoProcess->error()}, ERROR_ASSERT, 'cannot call error() after process has been closed');
}
}
1;

46
test/lib/pgBackRestTest/Module/Common/CommonLogPerlTest.pm
View File

@ -1,46 +0,0 @@
####################################################################################################################################
# CommonLogTest.pm - Unit tests for Log module
####################################################################################################################################
package pgBackRestTest::Module::Common::CommonLogPerlTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
################################################################################################################################
if ($self->begin("log()"))
{
logWarnOnErrorEnable();
$self->testResult(sub {&log(ERROR, "my test log", 27)}, "[object]", 'log error as warning',
{strLogExpect => "WARN: [027]: my test log"});
$self->testResult(sub {&log(INFO, "my test log")}, "my test log", 'log info as info',
{strLogLevel => INFO, strLogExpect => "INFO: my test log"});
logWarnOnErrorDisable();
$self->testResult(sub {&log(ERROR, "my test log", 27)}, "[object]", 'log error',
{strLogExpect => "ERROR: [027]: my test log"});
}
}
1;

249
test/lib/pgBackRestTest/Module/Info/InfoInfoArchivePerlTest.pm
View File

@ -1,249 +0,0 @@
####################################################################################################################################
# Unit tests for ArchiveInfo
####################################################################################################################################
package pgBackRestTest::Module::Info::InfoInfoArchivePerlTest;
use parent 'pgBackRestTest::Env::HostEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::Info;
use pgBackRest::Backup::Info;
use pgBackRest::Common::Cipher;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# initModule
####################################################################################################################################
sub initModule
{
my $self = shift;
$self->{strRepoPath} = $self->testPath() . '/repo';
}
####################################################################################################################################
# initTest
####################################################################################################################################
sub initTest
{
my $self = shift;
# Clear cache from the previous test
storageRepoCacheClear();
# Load options
$self->configTestClear();
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath() . '/repo');
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Create backup info path
storageRepo()->pathCreate(STORAGE_REPO_BACKUP, {bCreateParent => true});
# Create archive info path
storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE, {bCreateParent => true});
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
my $strArchiveTestFile = $self->dataPath() . '/backup.wal1_';
################################################################################################################################
if ($self->begin("Archive::Info::new()"))
{
$self->testException(sub {new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE))}, ERROR_FILE_MISSING,
ARCHIVE_INFO_FILE . " does not exist but is required to push/get WAL segments\n" .
"HINT: is archive_command configured in postgresql.conf?\n" .
"HINT: has a stanza-create been performed?\n" .
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.");
}
################################################################################################################################
if ($self->begin("Archive::Info::reconstruct()"))
{
my $tWalContent = $self->walGenerateContent(PG_VERSION_94);
my $strWalChecksum = $self->walGenerateContentChecksum(PG_VERSION_94);
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true});
storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE . "/" . PG_VERSION_94 . "-1/0000000100000001", {bCreateParent => true});
my $strArchiveFile = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/" . PG_VERSION_94 . "-1/0000000100000001/") .
"000000010000000100000001-${strWalChecksum}";
storageTest()->put($strArchiveFile, $tWalContent);
$self->testResult(sub {$oArchiveInfo->reconstruct(PG_VERSION_94, $self->dbSysId(PG_VERSION_94))}, "[undef]", 'reconstruct');
$self->testResult(sub {$oArchiveInfo->check(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), false)}, PG_VERSION_94 . "-1",
' check reconstruct');
# Attempt to reconstruct from an encypted archived WAL for an unencrypted repo
#---------------------------------------------------------------------------------------------------------------------------
# Prepend encryption Magic signature to simulate encryption
executeTest('echo "' . CIPHER_MAGIC . '" > ' . $strArchiveFile);
$self->testException(sub {$oArchiveInfo->reconstruct(PG_VERSION_94, $self->dbSysId(PG_VERSION_94))}, ERROR_CRYPTO,
"encryption incompatible for '$strArchiveFile'" .
"\nHINT: is or was the repo encrypted?");
executeTest('rm ' . $strArchiveFile);
# Attempt to reconstruct from an encypted archived WAL with an encrypted repo
#---------------------------------------------------------------------------------------------------------------------------
storageRepoCacheClear();
$self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
$self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, 'x');
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Instantiate an archive.info object with a sub passphrase for the archived WAL
my $strCipherPassSub = 'y';
$oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true, strCipherPassSub => $strCipherPassSub});
# Create an encrypted archived WAL
storageRepo()->put($strArchiveFile, $tWalContent, {strCipherPass => $strCipherPassSub});
$self->testResult(sub {$oArchiveInfo->reconstruct(PG_VERSION_94, $self->dbSysId(PG_VERSION_94))}, "[undef]", 'reconstruct');
$self->testResult(sub {$oArchiveInfo->check(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), false)}, PG_VERSION_94 . "-1",
' check reconstruction from encrypted archive');
$oArchiveInfo->save();
# Confirm encrypted
$self->testResult(sub {storageRepo()->encrypted(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE) . '/'
. ARCHIVE_INFO_FILE) && ($oArchiveInfo->cipherPassSub() eq $strCipherPassSub)}, true,
' new archive info encrypted');
}
################################################################################################################################
if ($self->begin("Archive::Info::archiveIdList(), check(), archiveId()"))
{
my @stryArchiveId;
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true});
$oArchiveInfo->create(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), false);
$oArchiveInfo->dbSectionSet(PG_VERSION_93, $self->dbSysId(PG_VERSION_93), $oArchiveInfo->dbHistoryIdGet(false) + 1);
$oArchiveInfo->dbSectionSet(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), $oArchiveInfo->dbHistoryIdGet(false) + 10);
$oArchiveInfo->dbSectionSet(PG_VERSION_93, $self->dbSysId(PG_VERSION_93), $oArchiveInfo->dbHistoryIdGet(false) + 1);
$oArchiveInfo->save();
# Check gets only the latest DB and returns only that archiveId
push(@stryArchiveId, $oArchiveInfo->check(PG_VERSION_93, $self->dbSysId(PG_VERSION_93)));
$self->testResult(sub {(@stryArchiveId == 1) && ($stryArchiveId[0] eq PG_VERSION_93 . "-13")}, true,
'check - return only newest archiveId');
$self->testResult(sub {$oArchiveInfo->archiveId(
{strDbVersion => PG_VERSION_93, ullDbSysId => $self->dbSysId(PG_VERSION_93)})},
PG_VERSION_93 . "-13", 'archiveId - return only newest archiveId for multiple histories');
$self->testException(
sub {$oArchiveInfo->archiveId({strDbVersion => PG_VERSION_94, ullDbSysId => BOGUS})}, ERROR_ARCHIVE_MISMATCH,
"unable to retrieve the archive id for database version '" . PG_VERSION_94 . "' and system-id '" . BOGUS . "'");
$self->testException(sub {$oArchiveInfo->check(PG_VERSION_94, $self->dbSysId(PG_VERSION_94))}, ERROR_ARCHIVE_MISMATCH,
"WAL segment version " . PG_VERSION_94 . " does not match archive version " . PG_VERSION_93 .
"\nWAL segment system-id " . $self->dbSysId(PG_VERSION_94) . " does not match archive system-id " .
$self->dbSysId(PG_VERSION_93) . "\nHINT: are you archiving to the correct stanza?");
@stryArchiveId = $oArchiveInfo->archiveIdList(PG_VERSION_93, $self->dbSysId(PG_VERSION_93));
$self->testResult(sub {(@stryArchiveId == 2) && ($stryArchiveId[0] eq PG_VERSION_93 . "-13") &&
($stryArchiveId[1] eq PG_VERSION_93 . "-2")}, true, 'archiveIdList - returns multiple archiveId - newest first');
@stryArchiveId = $oArchiveInfo->archiveIdList(PG_VERSION_94, $self->dbSysId(PG_VERSION_94));
$self->testResult(sub {(@stryArchiveId == 1) && ($stryArchiveId[0] eq PG_VERSION_94 . "-12")}, true,
'archiveIdList - returns older archiveId');
$self->testException(
sub {$oArchiveInfo->archiveIdList(PG_VERSION_95, $self->dbSysId(PG_VERSION_94))}, ERROR_ARCHIVE_MISMATCH,
"unable to retrieve the archive id for database version '" . PG_VERSION_95 . "' and system-id '" .
$self->dbSysId(PG_VERSION_94) . "'");
}
################################################################################################################################
if ($self->begin("encryption"))
{
# Create an unencrypted archive.info file
#---------------------------------------------------------------------------------------------------------------------------
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true});
$oArchiveInfo->create(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), true);
# Confirm unencrypted
$self->testResult(sub {storageRepo()->encrypted(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE) . '/'
. ARCHIVE_INFO_FILE)}, false, ' new archive info unencrypted');
my $strFile = $oArchiveInfo->{strFileName};
# Prepend encryption Magic signature to simulate encryption
executeTest('echo "' . CIPHER_MAGIC . '$(cat ' . $strFile . ')" > ' . $strFile);
$self->testException(sub {new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE))}, ERROR_CRYPTO,
"unable to parse '$strFile'" .
"\nHINT: is or was the repo encrypted?");
# Remove the archive info files
executeTest('rm ' . $oArchiveInfo->{strFileName} . '*');
# Create an encrypted storage and archive.info file
#---------------------------------------------------------------------------------------------------------------------------
my $strCipherPass = 'x';
$self->configTestClear();
$self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
$self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, $strCipherPass);
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath() . '/repo');
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Clear the storage repo settings
storageRepoCacheClear();
# Create an encrypted storage and generate an encryption sub passphrase to store in the file
my $strCipherPassSub = cipherPassGen();
# Error on encrypted repo but no passphrase passed to store in the file
$self->testException(sub {new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true})}, ERROR_ASSERT,
'a user passphrase and sub passphrase are both required when encrypting');
# Create an encrypted archiveInfo file
$oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true, strCipherPassSub => $strCipherPassSub});
$oArchiveInfo->create(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), true);
# Confirm encrypted
$self->testResult(sub {storageRepo()->encrypted(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE) . '/'
. ARCHIVE_INFO_FILE)}, true, ' new archive info encrypted');
$self->testResult(sub {$oArchiveInfo->test(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS, undef, $strCipherPassSub)},
true, ' generated passphrase stored');
}
}
1;

262
test/lib/pgBackRestTest/Module/Info/InfoInfoBackupPerlTest.pm
View File

@ -1,262 +0,0 @@
####################################################################################################################################
# Unit tests for BackupInfo
####################################################################################################################################
package pgBackRestTest::Module::Info::InfoInfoBackupPerlTest;
use parent 'pgBackRestTest::Env::HostEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::Info;
use pgBackRest::Backup::Info;
use pgBackRest::Common::Cipher;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# initModule
####################################################################################################################################
sub initModule
{
my $self = shift;
$self->{strRepoPath} = $self->testPath() . '/repo';
}
####################################################################################################################################
# initTest
####################################################################################################################################
sub initTest
{
my $self = shift;
# Load options
$self->configTestClear();
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath() . '/repo');
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Create backup info path
storageRepo()->pathCreate(STORAGE_REPO_BACKUP, {bCreateParent => true});
# Create archive info path
storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE, {bCreateParent => true});
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
my $i93ControlVersion = 937;
my $i93CatalogVersion = 201306121;
my $i94ControlVersion = 942;
my $i94CatalogVersion = 201409291;
################################################################################################################################
if ($self->begin("BackupInfo::check() && BackupInfo::confirmDb()"))
{
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP), false, false,
{bIgnoreMissing => true});
$oBackupInfo->create(PG_VERSION_93, $self->dbSysId(PG_VERSION_93), $i93ControlVersion, $i93CatalogVersion, true);
# All DB section matches
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oBackupInfo->check(
PG_VERSION_93, $i93ControlVersion, $i93CatalogVersion, $self->dbSysId(PG_VERSION_93), false)}, 1, 'db section matches');
# DB section version mismatch
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oBackupInfo->check(PG_VERSION_94, $i93ControlVersion, $i93CatalogVersion,
$self->dbSysId(PG_VERSION_93))}, ERROR_BACKUP_MISMATCH,
"database version = " . &PG_VERSION_94 . ", system-id " . $self->dbSysId(PG_VERSION_93) .
" does not match backup version = " . &PG_VERSION_93 . ", " . "system-id = " . $self->dbSysId(PG_VERSION_93) .
"\nHINT: is this the correct stanza?");
# DB section system-id mismatch
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$oBackupInfo->check(PG_VERSION_93, $i93ControlVersion, $i93CatalogVersion, $self->dbSysId(PG_VERSION_94))},
ERROR_BACKUP_MISMATCH,
"database version = " . &PG_VERSION_93 . ", system-id " . $self->dbSysId(PG_VERSION_94) .
" does not match backup version = " . &PG_VERSION_93 . ", " . "system-id = " . $self->dbSysId(PG_VERSION_93) .
"\nHINT: is this the correct stanza?");
# DB section control version mismatch
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oBackupInfo->check(PG_VERSION_93, 123, $i93CatalogVersion, $self->dbSysId(PG_VERSION_93))},
ERROR_BACKUP_MISMATCH, "database control-version = 123, catalog-version $i93CatalogVersion " .
"does not match backup control-version = $i93ControlVersion, catalog-version = $i93CatalogVersion" .
"\nHINT: this may be a symptom of database or repository corruption!");
# DB section catalog version mismatch
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oBackupInfo->check(PG_VERSION_93, $i93ControlVersion, 123456789, $self->dbSysId(PG_VERSION_93))},
ERROR_BACKUP_MISMATCH, "database control-version = $i93ControlVersion, catalog-version 123456789 " .
"does not match backup control-version = $i93ControlVersion, catalog-version = $i93CatalogVersion" .
"\nHINT: this may be a symptom of database or repository corruption!");
my $strBackupLabel = "20170403-175647F";
$oBackupInfo->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID,
$oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID));
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oBackupInfo->confirmDb($strBackupLabel, PG_VERSION_93, $self->dbSysId(PG_VERSION_93))}, true,
'backup db matches');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oBackupInfo->confirmDb($strBackupLabel, PG_VERSION_94, $self->dbSysId(PG_VERSION_93))}, false,
'backup db wrong version');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oBackupInfo->confirmDb($strBackupLabel, PG_VERSION_93, $self->dbSysId(PG_VERSION_94))}, false,
'backup db wrong system-id');
}
################################################################################################################################
if ($self->begin("BackupInfo::backupArchiveDbHistoryId()"))
{
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP), false, false,
{bIgnoreMissing => true});
$oBackupInfo->create(PG_VERSION_93, $self->dbSysId(PG_VERSION_93), $i93ControlVersion, $i93CatalogVersion, true);
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true});
$oArchiveInfo->create(PG_VERSION_93, $self->dbSysId(PG_VERSION_93), true);
# Map archiveId to Backup history id
#---------------------------------------------------------------------------------------------------------------------------
my $strArchiveId = $oArchiveInfo->archiveId({strDbVersion => PG_VERSION_93, ullDbSysId => $self->dbSysId(PG_VERSION_93)});
$self->testResult(sub {$oBackupInfo->backupArchiveDbHistoryId($strArchiveId,
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE))}, 1, 'backupArchiveDbHistoryId found');
# Unable to map archiveId to Backup history id
#---------------------------------------------------------------------------------------------------------------------------
$strArchiveId = &PG_VERSION_94 . "-2";
$self->testResult(sub {$oBackupInfo->backupArchiveDbHistoryId($strArchiveId,
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE))}, "[undef]", 'backupArchiveDbHistoryId not found');
# Same version but different ullsystemid so continue looking until find match
#---------------------------------------------------------------------------------------------------------------------------
# Update db section and db history sections
my $iHistoryId = $oBackupInfo->dbHistoryIdGet(true)+1;
$oBackupInfo->dbSectionSet(
PG_VERSION_94, $i94ControlVersion, $i94CatalogVersion, $self->dbSysId(PG_VERSION_93), $iHistoryId);
$oArchiveInfo->dbSectionSet(PG_VERSION_94, $self->dbSysId(PG_VERSION_93), $iHistoryId);
$oBackupInfo->save();
$oArchiveInfo->save();
$strArchiveId = &PG_VERSION_94 . "-" . $iHistoryId;
$self->testResult(sub {$oBackupInfo->backupArchiveDbHistoryId($strArchiveId,
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE))}, $iHistoryId, 'same db version but different system-id');
# Different version but same ullsystemid
#---------------------------------------------------------------------------------------------------------------------------
$iHistoryId = $oBackupInfo->dbHistoryIdGet(false)+1;
$oBackupInfo->dbSectionSet(
PG_VERSION_94, $i93ControlVersion, $i93CatalogVersion, $self->dbSysId(PG_VERSION_94), $iHistoryId);
$oArchiveInfo->dbSectionSet(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), $iHistoryId);
$oBackupInfo->save();
$oArchiveInfo->save();
$strArchiveId = &PG_VERSION_93 . "-" . $iHistoryId;
$self->testResult(sub {$oBackupInfo->backupArchiveDbHistoryId($strArchiveId,
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE))}, "[undef]", 'same db system-id but different version');
# First and last version and ullsystemid same in backup.info but only 1st in archive info - return last
#---------------------------------------------------------------------------------------------------------------------------
$iHistoryId = $oBackupInfo->dbHistoryIdGet(true)+1;
$oBackupInfo->dbSectionSet(
PG_VERSION_93, $i93ControlVersion, $i93CatalogVersion, $self->dbSysId(PG_VERSION_93), $iHistoryId);
$oBackupInfo->save();
$strArchiveId = &PG_VERSION_93 . "-1";
$self->testResult(sub {$oBackupInfo->backupArchiveDbHistoryId($strArchiveId,
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE))}, $iHistoryId, 'duplicate 1st and last version/sysid - last chosen');
}
################################################################################################################################
if ($self->begin("encryption"))
{
# Create a backupInfo file
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP), false, false,
{bIgnoreMissing => true});
$oBackupInfo->create(PG_VERSION_93, $self->dbSysId(PG_VERSION_93), $i93ControlVersion, $i93CatalogVersion, true);
my $strFile = $oBackupInfo->{strFileName};
# Prepend encryption Magic signature to simulate encryption
executeTest('echo "' . CIPHER_MAGIC . '$(cat ' . $strFile . ')" > ' . $strFile);
$self->testException(sub {new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP))}, ERROR_CRYPTO,
"unable to parse '$strFile'" .
"\nHINT: is or was the repo encrypted?");
# Create encrypted files, change the passphrase and attempt to load - ensure flush error returned as parse error
#---------------------------------------------------------------------------------------------------------------------------
# Remove the backup info files
executeTest('rm ' . $oBackupInfo->{strFileName} . '*');
# Clear the storage repo settings and change the passphrase
storageRepoCacheClear();
my $strCipherPass = 'x';
$self->configTestClear();
$self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
$self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, $strCipherPass);
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath() . '/repo');
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Confirm exception when not passing a sub passphrase
$self->testException(sub {new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP), false, false,
{bIgnoreMissing => true})}, ERROR_ASSERT,
'a user passphrase and sub passphrase are both required when encrypting');
my $strCipherPassSub = cipherPassGen();
# Create encrypted files
$oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP), false, false,
{bIgnoreMissing => true, strCipherPassSub => $strCipherPassSub});
$oBackupInfo->create(PG_VERSION_93, $self->dbSysId(PG_VERSION_93), $i93ControlVersion, $i93CatalogVersion, true);
# Clear the storage repo settings and change the passphrase
storageRepoCacheClear();
$self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
$self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, BOGUS);
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath() . '/repo');
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testException(sub {new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP))}, ERROR_CRYPTO,
"unable to parse '" . $oBackupInfo->{strFileName} . "'" .
"\nHINT: is or was the repo encrypted?");
}
}
1;

1733
test/lib/pgBackRestTest/Module/Manifest/ManifestAllPerlTest.pm
View File

File diff suppressed because it is too large Load Diff

3
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
View File

@ -26,6 +26,7 @@ use pgBackRest::InfoCommon;
use pgBackRest::LibC qw(:checksum);
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
@ -142,7 +143,7 @@ sub run
'184473f470864e067ee3a22e64b47b0a1c356f29', $lTime, undef, true);
# Load sample page
my $tBasePage = ${storageTest()->get($self->dataPath() . '/page.bin')};
my $tBasePage = ${storageLocal()->get($self->dataPath() . '/page.bin')};
my $iBasePageChecksum = 0x1B99;
# Create base path

135
test/lib/pgBackRestTest/Module/Protocol/ProtocolCommonMinionPerlTest.pm
View File

@ -1,135 +0,0 @@
####################################################################################################################################
# Tests for Protocol::Common::Minion module
####################################################################################################################################
package pgBackRestTest::Module::Protocol::ProtocolCommonMinionPerlTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use IO::Socket::UNIX;
use Time::HiRes qw(usleep);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::LibC qw(:config);
use pgBackRest::Protocol::Base::Minion;
use pgBackRest::Version;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# socketServer
####################################################################################################################################
sub socketServer
{
my $self = shift;
my $strSocketFile = shift;
my $fnServer = shift;
# Fork off the server
if (fork() == 0)
{
# Open the domain socket
my $oSocketServer = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Local => $strSocketFile, Listen => 1);
&log(INFO, " * socket server open");
# Wait for a connection and create IO object
my $oConnection = $oSocketServer->accept();
my $oIoHandle = new pgBackRest::Common::Io::Handle('socket server', $oConnection, $oConnection);
&log(INFO, " * socket server connected");
# Run server function
$fnServer->($oIoHandle);
# Shutdown server
$oSocketServer->close();
&log(INFO, " * socket server closed");
unlink($strSocketFile);
exit 0;
}
# Wait for client socket
my $oWait = waitInit(5);
while (!-e $strSocketFile && waitMore($oWait)) {};
# Open the client socket
my $oClient = IO::Socket::UNIX->new(Type => SOCK_STREAM(), Peer => $strSocketFile);
if (!defined($oClient))
{
logErrorResult(ERROR_FILE_OPEN, 'unable to open client socket', $OS_ERROR);
}
&log(INFO, " * socket client connected");
return $oClient;
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Test data
my $strSocketFile = $self->testPath() . qw{/} . 'domain.socket';
################################################################################################################################
# if ($self->begin('new() & timeout() & bufferMax()'))
# {
# #---------------------------------------------------------------------------------------------------------------------------
# my $oIoBuffered = $self->testResult(
# sub {new pgBackRest::Common::Io::Buffered(
# new pgBackRest::Common::Io::Handle('test'), 10, 2048)}, '[object]', 'new - no handles');
#
# $self->testResult(sub {$oIoBuffered->timeout()}, 10, ' check timeout');
# $self->testResult(sub {$oIoBuffered->bufferMax()}, 2048, ' check buffer max');
# }
################################################################################################################################
if ($self->begin('process()'))
{
my $oClient = $self->socketServer($strSocketFile, sub
{
my $oIoHandle = shift;
my $oMinion = new pgBackRest::Protocol::Base::Minion('test', new pgBackRest::Common::Io::Buffered($oIoHandle, 5, 4096));
# Use bogus lock path to ensure a lock is not taken for the archive-get command
$oMinion->process($self->testPath(), cfgCommandName(CFGCMD_ARCHIVE_GET), "test", 0);
});
#---------------------------------------------------------------------------------------------------------------------------
my $oIoBuffered = $self->testResult(
sub {new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle('socket client', $oClient, $oClient), 5, 4096)}, '[object]', 'open');
$self->testResult(
sub {$oIoBuffered->readLine()},
'{"name":"' . PROJECT_NAME . '","service":"test","version":"' . PROJECT_VERSION . '"}', 'read greeting');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oIoBuffered->writeLine('{"cmd":"noop"}')}, 15, 'write noop');
$self->testResult(
sub {$oIoBuffered->readLine()}, '{"out":[]}', 'read output');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oIoBuffered->writeLine('{"cmd":"exit"}')}, 15, 'write exit');
$self->testResult(
sub {$oIoBuffered->readLine(true)}, undef, 'read EOF');
}
}
1;

155
test/lib/pgBackRestTest/Module/Protocol/ProtocolHelperPerlTest.pm
View File

@ -1,155 +0,0 @@
####################################################################################################################################
# Protocol Helper Tests
####################################################################################################################################
package pgBackRestTest::Module::Protocol::ProtocolHelperPerlTest;
use parent 'pgBackRestTest::Env::ConfigEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::Common;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# initModule
####################################################################################################################################
sub initModule
{
my $self = shift;
$self->{strDbPath} = $self->testPath() . '/db';
$self->{strRepoPath} = $self->testPath() . '/repo';
$self->{strArchivePath} = "$self->{strRepoPath}/archive/" . $self->stanza();
}
####################################################################################################################################
# initTest
####################################################################################################################################
sub initTest
{
my $self = shift;
# Create archive info
storageTest()->pathCreate($self->{strArchivePath}, {bIgnoreExists => true, bCreateParent => true});
}
####################################################################################################################################
# initOption
####################################################################################################################################
sub initOption
{
my $self = shift;
$self->configTestClear();
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_PG_PATH, $self->{strDbPath});
$self->optionTestSet(CFGOPT_REPO_PATH, $self->{strRepoPath});
$self->optionTestSet(CFGOPT_LOG_PATH, $self->testPath());
$self->optionTestSetBool(CFGOPT_COMPRESS, false);
$self->optionTestSet(CFGOPT_DB_TIMEOUT, 5);
$self->optionTestSet(CFGOPT_PROTOCOL_TIMEOUT, 6);
$self->optionTestSet(CFGOPT_ARCHIVE_TIMEOUT, 3);
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
################################################################################################################################
if ($self->begin('protocolParam()'))
{
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_HOST, 1), 'pg-host-1');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_PATH, 1), '/db1');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_PORT, 1), '1111');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, 1), 'pgbackrest1');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_HOST, 2), 'pg-host-2');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_PATH, 2), '/db2');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_PORT, 2), '2222');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, 2), 'pgbackrest2');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, 2), '/config2');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH, 2), '/config-include2');
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG_PATH, 2), '/config-path2');
$self->configTestLoad(CFGCMD_BACKUP);
$self->testResult(
sub {pgBackRest::Protocol::Helper::protocolParam(cfgCommandName(CFGCMD_BACKUP), CFGOPTVAL_REMOTE_TYPE_DB, 2)},
'(pg-host-2, postgres, [undef], pgbackrest2 --buffer-size=4194304 --command=backup --compress-level=6' .
' --compress-level-network=3 --config=/config2 --config-include-path=/config-include2 --config-path=/config-path2' .
' --log-level-file=off --pg1-path=/db2 --pg1-port=2222 --process=0 --protocol-timeout=1830 --stanza=db --type=db' .
' remote)',
'more than one backup db host');
# --------------------------------------------------------------------------------------------------------------------------
$self->configTestClear();
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(cfgOptionIdFromIndex(CFGOPT_PG_PATH, 1), '/db1');
$self->optionTestSet(CFGOPT_REPO_HOST, 'repo-host');
$self->optionTestSet(CFGOPT_REPO_PATH, '/repo');
$self->optionTestSet(CFGOPT_REPO_HOST_CMD, 'pgbackrest-repo');
$self->optionTestSet(CFGOPT_REPO_HOST_CONFIG, '/config-repo');
$self->optionTestSet(CFGOPT_REPO_HOST_CONFIG_INCLUDE_PATH, '/config-include-repo');
$self->optionTestSet(CFGOPT_REPO_HOST_CONFIG_PATH, '/config-path-repo');
$self->configTestLoad(CFGCMD_RESTORE);
$self->testResult(
sub {pgBackRest::Protocol::Helper::protocolParam(cfgCommandName(CFGCMD_RESTORE), CFGOPTVAL_REMOTE_TYPE_BACKUP)},
'(repo-host, pgbackrest, [undef], pgbackrest-repo --buffer-size=4194304 --command=restore --compress-level=6' .
' --compress-level-network=3 --config=/config-repo --config-include-path=/config-include-repo' .
' --config-path=/config-path-repo --log-level-file=off --pg1-path=/db1 --process=0 --protocol-timeout=1830' .
' --repo1-path=/repo --stanza=db --type=backup remote)',
'config params to repo host');
}
################################################################################################################################
if ($self->begin("Protocol::Helper"))
{
$self->initOption();
$self->optionTestSet(CFGOPT_REPO_HOST, 'localhost');
$self->optionTestSet(CFGOPT_REPO_HOST_USER, $self->pgUser());
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testResult(
sub {protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP, undef, {strBackRestBin => $self->backrestExe()})}, "[object]",
'ssh default port');
# Destroy protocol object
protocolDestroy();
$self->optionTestSet(CFGOPT_REPO_HOST_PORT, 25);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testException(
sub {protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP, undef, {strBackRestBin => $self->backrestExe()})}, ERROR_FILE_READ,
"remote process on 'localhost' terminated unexpectedly: ssh: connect to host localhost port 25:");
# Destroy protocol object
protocolDestroy();
}
}
1;

13
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
View File

@ -25,6 +25,7 @@ use pgBackRest::Config::Config;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
@ -319,7 +320,7 @@ sub run
'fail on backup info file missing from non-empty dir', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
# Change the database version by copying a new pg_control file to a new pg-path to use for db mismatch test
storageDb()->pathCreate(
storageLocal()->pathCreate(
$oHostDbMaster->dbPath() . '/testbase/' . DB_PATH_GLOBAL,
{strMode => '0700', bIgnoreExists => true, bCreateParent => true});
$self->controlGenerate(
@ -769,7 +770,7 @@ sub run
# Version <= 8.4 always places a PG_VERSION file in the tablespace
if ($oHostDbMaster->pgVersion() <= PG_VERSION_84)
{
if (!storageDb()->exists("${strTablespacePath}/" . DB_FILE_PGVERSION))
if (!storageLocal()->exists("${strTablespacePath}/" . DB_FILE_PGVERSION))
{
confess &log(ASSERT, "unable to find '" . DB_FILE_PGVERSION . "' in tablespace path '${strTablespacePath}'");
}
@ -787,14 +788,14 @@ sub run
'/PG_' . $oHostDbMaster->pgVersion() . qw{_} . $oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG);
# Check that path exists
if (!storageDb()->pathExists($strTablespacePath))
if (!storageLocal()->pathExists($strTablespacePath))
{
confess &log(ASSERT, "unable to find tablespace path '${strTablespacePath}'");
}
}
# Make sure there are some files in the tablespace path (exclude PG_VERSION if <= 8.4 since that was tested above)
if (grep(!/^PG\_VERSION$/i, storageDb()->list($strTablespacePath)) == 0)
if (grep(!/^PG\_VERSION$/i, storageLocal()->list($strTablespacePath)) == 0)
{
confess &log(ASSERT, "no files found in tablespace path '${strTablespacePath}'");
}
@ -864,7 +865,7 @@ sub run
# Save recovery file to test so we can use it in the next test
$strRecoveryFile = $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'postgresql.auto.conf' : DB_FILE_RECOVERYCONF;
storageDb()->copy(
storageLocal()->copy(
$oHostDbMaster->dbBasePath() . qw{/} . $strRecoveryFile, $self->testPath() . qw{/} . $strRecoveryFile);
$oHostDbMaster->clusterStart();
@ -886,7 +887,7 @@ sub run
executeTest('rm -rf ' . $oHostDbMaster->tablespacePath(1) . "/*");
# Restore recovery file that was saved in last test
storageDb()->move($self->testPath . "/${strRecoveryFile}", $oHostDbMaster->dbBasePath() . "/${strRecoveryFile}");
storageLocal()->move($self->testPath . "/${strRecoveryFile}", $oHostDbMaster->dbBasePath() . "/${strRecoveryFile}");
# Also touch recovery.signal when required
if ($oHostDbMaster->pgVersion() >= PG_VERSION_12)

116
test/lib/pgBackRestTest/Module/Storage/StorageHelperPerlTest.pm
View File

@ -1,116 +0,0 @@
####################################################################################################################################
# StorageHelperTest.pm - Tests for Storage::Helper module.
####################################################################################################################################
package pgBackRestTest::Module::Storage::StorageHelperPerlTest;
use parent 'pgBackRestTest::Env::ConfigEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Storable qw(dclone);
use pgBackRest::Config::Config;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# initTest - initialization before each test
####################################################################################################################################
sub initTest
{
my $self = shift;
storageTest()->pathCreate('db');
storageTest()->pathCreate('repo');
storageTest()->pathCreate('spool');
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Define test file
my $strFile = 'file.txt';
my $strFileCopy = 'file.txt.copy';
my $strFileContent = 'TESTDATA';
my $iFileSize = length($strFileContent);
# Setup parameters
$self->optionTestSet(CFGOPT_PG_PATH, $self->testPath() . '/db');
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath() . '/repo');
$self->optionTestSet(CFGOPT_SPOOL_PATH, $self->testPath() . '/spool');
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSetBool(CFGOPT_ARCHIVE_ASYNC, true);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
#-------------------------------------------------------------------------------------------------------------------------------
if ($self->begin("storageLocal()"))
{
$self->testResult(sub {storageLocal($self->testPath())->put("/tmp/${strFile}", $strFileContent)}, $iFileSize, 'put');
$self->testResult(sub {${storageTest()->get("/tmp/${strFile}")}}, $strFileContent, ' check put');
$self->testResult(
sub {storageLocal($self->testPath())->put("/tmp/${strFile}", $strFileContent)}, $iFileSize, 'put cache storage');
$self->testResult(sub {${storageTest()->get("/tmp/${strFile}")}}, $strFileContent, ' check put');
}
#-------------------------------------------------------------------------------------------------------------------------------
if ($self->begin("storageDb()"))
{
$self->testResult(sub {storageDb()->put($strFile, $strFileContent)}, $iFileSize, 'put');
$self->testResult(sub {${storageTest()->get("db/${strFile}")}}, $strFileContent, ' check put');
$self->testResult(sub {storageDb()->put($strFileCopy, $strFileContent)}, $iFileSize, 'put cached storage');
$self->testResult(sub {${storageTest()->get("db/${strFileCopy}")}}, $strFileContent, ' check put');
}
#-------------------------------------------------------------------------------------------------------------------------------
if ($self->begin("storageRepo()"))
{
$self->testResult(sub {storageRepo()->put($strFile, $strFileContent)}, $iFileSize, 'put');
$self->testResult(sub {${storageTest()->get("repo/${strFile}")}}, $strFileContent, ' check put');
$self->testResult(sub {storageRepo()->put($strFileCopy, $strFileContent)}, $iFileSize, 'put cached storage');
$self->testResult(sub {${storageTest()->get("repo/${strFileCopy}")}}, $strFileContent, ' check put');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE)}, $self->testPath() . '/repo/archive/db', 'check archive path');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . '/9.3-1/000000010000000100000001')},
$self->testPath() . '/repo/archive/db/9.3-1/0000000100000001/000000010000000100000001', 'check repo WAL file');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . '/9.3-1/0000000100000001')},
$self->testPath() . '/repo/archive/db/9.3-1/0000000100000001', 'check repo WAL path');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . '/9.3-1/0000000100000001')},
$self->testPath() . '/repo/archive/db/9.3-1/0000000100000001', 'check repo WAL major path');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . '/9.3-1')},
$self->testPath() . '/repo/archive/db/9.3-1', 'check repo archive id path');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_BACKUP)}, $self->testPath() . '/repo/backup/db', 'check backup path');
$self->testResult(
sub {storageRepo()->pathGet(STORAGE_REPO_BACKUP . '/file')}, $self->testPath() . '/repo/backup/db/file',
'check backup file');
}
}
1;

351
test/lib/pgBackRestTest/Module/Storage/StoragePerlTest.pm
View File

@ -1,351 +0,0 @@
####################################################################################################################################
# Tests for Storage::Local module
####################################################################################################################################
package pgBackRestTest::Module::Storage::StoragePerlTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Config::Config;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::LibC qw(:crypto);
use pgBackRest::Storage::Base;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Env::Host::HostBackupTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Define test file
my $strFile = $self->testPath() . '/file.txt';
my $strFileCopy = $self->testPath() . '/file.txt.copy';
my $strFileHash = 'bbbcf2c59433f68f22376cd2439d6cd309378df6';
my $strFileContent = 'TESTDATA';
my $iFileSize = length($strFileContent);
# Create local storage
$self->{oStorageLocal} = new pgBackRest::Storage::Storage('<LOCAL>');
################################################################################################################################
if ($self->begin("pathGet()"))
{
$self->testResult(sub {$self->storageLocal()->pathGet('file')}, '/file', 'relative path');
$self->testResult(sub {$self->storageLocal()->pathGet('/file2')}, '/file2', 'absolute path');
}
################################################################################################################################
if ($self->begin('put()'))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile))}, 0, 'put empty');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($strFile)}, 0, 'put empty (all defaults)');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), $strFileContent)}, $iFileSize, 'put');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->put($self->storageLocal()->openWrite($strFile), \$strFileContent)}, $iFileSize,
'put reference');
}
################################################################################################################################
if ($self->begin('get()'))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->get($self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}))}, undef,
'get missing');
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->put($strFile);
$self->testResult(sub {${$self->storageLocal()->get($strFile)}}, undef, 'get empty');
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->put($strFile, $strFileContent);
$self->testResult(sub {${$self->storageLocal()->get($strFile)}}, $strFileContent, 'get');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {${$self->storageLocal()->get($self->storageLocal()->openRead($strFile))}}, $strFileContent, 'get from io');
}
################################################################################################################################
if ($self->begin('hashSize()'))
{
$self->testResult(
sub {$self->storageLocal()->put($strFile, $strFileContent)}, 8, 'put');
$self->testResult(
sub {$self->storageLocal()->hashSize($strFile)},
qw{(} . cryptoHashOne('sha1', $strFileContent) . ', ' . $iFileSize . qw{)}, ' check hash/size');
$self->testResult(
sub {$self->storageLocal()->hashSize(BOGUS, {bIgnoreMissing => true})}, "([undef], [undef])",
' check missing hash/size');
}
################################################################################################################################
if ($self->begin('copy()'))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING,
"unable to open missing file '${strFile}' for read");
$self->testResult(
sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$self->storageLocal()->copy(
$self->storageLocal()->openRead($strFile, {bIgnoreMissing => true}),
$self->storageLocal()->openWrite($strFileCopy))},
false, 'missing source io');
$self->testResult(
sub {$self->storageLocal()->exists($strFileCopy)}, false, ' destination does not exist');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, ERROR_FILE_MISSING,
"unable to open missing file '${strFile}' for read");
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->put($strFile, $strFileContent);
$self->testResult(sub {$self->storageLocal()->copy($strFile, $strFileCopy)}, true, 'copy filename->filename');
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->remove($strFileCopy);
$self->testResult(
sub {$self->storageLocal()->copy($self->storageLocal()->openRead($strFile), $strFileCopy)}, true, 'copy io->filename');
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
#---------------------------------------------------------------------------------------------------------------------------
$self->storageLocal()->remove($strFileCopy);
$self->testResult(
sub {$self->storageLocal()->copy(
$self->storageLocal()->openRead($strFile), $self->storageLocal()->openWrite($strFileCopy))},
true, 'copy io->io');
$self->testResult(sub {${$self->storageLocal()->get($strFileCopy)}}, $strFileContent, ' check copy');
}
################################################################################################################################
if ($self->begin('exists()'))
{
$self->storageLocal()->put($self->testPath() . "/test.file");
$self->testResult(sub {$self->storageLocal()->exists($self->testPath() . "/test.file")}, true, 'existing file');
$self->testResult(sub {$self->storageLocal()->exists($self->testPath() . "/test.missing")}, false, 'missing file');
$self->testResult(sub {$self->storageLocal()->exists($self->testPath())}, false, 'path');
}
################################################################################################################################
if ($self->begin('info()'))
{
$self->testResult(
sub {$self->storageLocal()->info($self->testPath())},
"{group => " . $self->group() . ", mode => 0770, type => d, user => " . $self->pgUser() . "}",
'stat dir successfully');
$self->testException(sub {$self->storageLocal()->info(BOGUS)}, ERROR_FILE_OPEN,
"unable to get info for missing path/file '/bogus'");
}
################################################################################################################################
if ($self->begin("manifest() and list()"))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->manifest($self->testPath() . '/missing')},
ERROR_PATH_MISSING, "unable to list file info for missing path '" . $self->testPath() . "/missing'");
#---------------------------------------------------------------------------------------------------------------------------
# Setup test data
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1');
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub1/sub2');
executeTest('mkdir -m 750 ' . $self->testPath() . '/sub2');
executeTest("echo 'TESTDATA' > " . $self->testPath() . '/test.txt');
utime(1111111111, 1111111111, $self->testPath() . '/test.txt');
executeTest('chmod 1640 ' . $self->testPath() . '/test.txt');
executeTest("echo 'TESTDATA_' > ". $self->testPath() . '/sub1/test-sub1.txt');
utime(1111111112, 1111111112, $self->testPath() . '/sub1/test-sub1.txt');
executeTest('chmod 0640 ' . $self->testPath() . '/sub1/test-sub1.txt');
executeTest("echo 'TESTDATA_' > ". $self->testPath() . '/sub1/.test-sub1.txt');
utime(1111111112, 1111111112, $self->testPath() . '/sub1/.test-sub1.txt');
executeTest('chmod 0640 ' . $self->testPath() . '/sub1/.test-sub1.txt');
executeTest("echo 'TESTDATA__' > " . $self->testPath() . '/sub1/sub2/test-sub2.txt');
utime(1111111113, 1111111113, $self->testPath() . '/sub1/sub2/test-sub2.txt');
executeTest('chmod 0646 ' . $self->testPath() . '/sub1/sub2/test-sub2.txt');
executeTest('mkfifo -m 606 ' . $self->testPath() . '/sub1/apipe');
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/test-hardlink.txt');
executeTest('ln ' . $self->testPath() . '/test.txt ' . $self->testPath() . '/sub1/sub2/test-hardlink.txt');
executeTest('ln -s .. ' . $self->testPath() . '/sub1/test');
executeTest('chmod 0700 ' . $self->testPath() . '/sub1/test');
executeTest('ln -s ../.. ' . $self->testPath() . '/sub1/sub2/test');
executeTest('chmod 0750 ' . $self->testPath() . '/sub1/sub2/test');
executeTest('chmod 0770 ' . $self->testPath());
$self->testResult(
sub {$self->storageLocal()->manifest($self->testPath())},
'{. => {group => ' . $self->group() . ', mode => 0770, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1/.test-sub1.txt => ' .
'{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111112, size => 10, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/apipe => {group => ' . $self->group() . ', mode => 0606, type => s, user => ' . $self->pgUser() . '}, ' .
'sub1/sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'sub1/sub2/test => {group => ' . $self->group() . ', link_destination => ../.., type => l, user => ' .
$self->pgUser() . '}, ' .
'sub1/sub2/test-hardlink.txt => ' .
'{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/sub2/test-sub2.txt => ' .
'{group => ' . $self->group() . ', mode => 0646, modification_time => 1111111113, size => 11, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/test => {group => ' . $self->group() . ', link_destination => .., type => l, user => ' . $self->pgUser() . '}, ' .
'sub1/test-hardlink.txt => ' .
'{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub1/test-sub1.txt => ' .
'{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111112, size => 10, type => f, user => ' .
$self->pgUser() . '}, ' .
'sub2 => {group => ' . $self->group() . ', mode => 0750, type => d, user => ' . $self->pgUser() . '}, ' .
'test.txt => ' .
'{group => ' . $self->group() . ', mode => 0640, modification_time => 1111111111, size => 9, type => f, user => ' .
$self->pgUser() . '}}',
'complete manifest');
$self->testResult(sub {$self->storageLocal()->list($self->testPath())}, "(sub1, sub2, test.txt)", "list");
$self->testResult(sub {$self->storageLocal()->list($self->testPath(), {strExpression => "2\$"})}, "sub2", "list");
$self->testResult(
sub {$self->storageLocal()->list($self->testPath(), {strSortOrder => 'reverse'})}, "(test.txt, sub2, sub1)",
"list reverse");
$self->testResult(sub {$self->storageLocal()->list($self->testPath() . "/sub2")}, "[undef]", "list empty");
$self->testResult(
sub {$self->storageLocal()->list($self->testPath() . "/sub99", {bIgnoreMissing => true})}, "[undef]", "list missing");
$self->testException(
sub {$self->storageLocal()->list($self->testPath() . "/sub99")}, ERROR_PATH_MISSING,
"unable to list files for missing path '" . $self->testPath() . "/sub99'");
}
################################################################################################################################
if ($self->begin('move()'))
{
my $strFileCopy = "${strFile}.copy";
my $strFileSub = $self->testPath() . '/sub/file.txt';
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {$self->storageLocal()->move($strFile, $strFileCopy)}, ERROR_FILE_MOVE,
"unable to move '${strFile}' to '${strFile}.copy': No such file or directory");
}
################################################################################################################################
if ($self->begin('owner()'))
{
my $strFile = $self->testPath() . "/test.txt";
$self->testException(
sub {$self->storageLocal()->owner($strFile, 'root')}, ERROR_FILE_MISSING,
"unable to stat '${strFile}': No such file or directory");
executeTest("touch ${strFile}");
$self->testException(
sub {$self->storageLocal()->owner($strFile, BOGUS)}, ERROR_FILE_OWNER,
"unable to set ownership for '${strFile}' because user 'bogus' does not exist");
$self->testException(
sub {$self->storageLocal()->owner($strFile, undef, BOGUS)}, ERROR_FILE_OWNER,
"unable to set ownership for '${strFile}' because group 'bogus' does not exist");
$self->testResult(sub {$self->storageLocal()->owner($strFile)}, undef, "no ownership changes");
$self->testResult(sub {$self->storageLocal()->owner($strFile, TEST_USER)}, undef, "same user");
$self->testResult(sub {$self->storageLocal()->owner($strFile, undef, TEST_GROUP)}, undef, "same group");
$self->testResult(
sub {$self->storageLocal()->owner($strFile, TEST_USER, TEST_GROUP)}, undef,
"same user, group");
$self->testException(
sub {$self->storageLocal()->owner($strFile, 'root', undef)}, ERROR_FILE_OWNER,
"unable to set ownership for '${strFile}': Operation not permitted");
$self->testException(
sub {$self->storageLocal()->owner($strFile, undef, 'root')}, ERROR_FILE_OWNER,
"unable to set ownership for '${strFile}': Operation not permitted");
executeTest("sudo chown :root ${strFile}");
$self->testResult(
sub {$self->storageLocal()->owner($strFile, undef, TEST_GROUP)}, undef, "change group back from root");
}
################################################################################################################################
if ($self->begin('pathCreate()'))
{
my $strTestPath = $self->testPath() . "/" . BOGUS;
$self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath)}, "[undef]",
"test creation of path " . $strTestPath);
$self->testException(sub {$self->storageLocal()->pathCreate($strTestPath)}, ERROR_PATH_CREATE,
"unable to create path '". $strTestPath. "'");
$self->testResult(sub {$self->storageLocal()->pathCreate($strTestPath, {bIgnoreExists => true})}, "[undef]",
"ignore path exists");
}
################################################################################################################################
if ($self->begin('pathExists()'))
{
$self->storageLocal()->put($self->testPath() . "/test.file");
$self->testResult(sub {$self->storageLocal()->pathExists($self->testPath() . "/test.file")}, false, 'existing file');
$self->testResult(sub {$self->storageLocal()->pathExists($self->testPath() . "/test.missing")}, false, 'missing file');
$self->testResult(sub {$self->storageLocal()->pathExists($self->testPath())}, true, 'path');
}
################################################################################################################################
if ($self->begin('pathSync()'))
{
$self->testResult(sub {$self->storageLocal()->pathSync($self->testPath())}, "[undef]", "test path sync");
}
}
####################################################################################################################################
# Getters
####################################################################################################################################
# sub host {return '127.0.0.1'}
# sub pathLocal {return shift->{strPathLocal}};
# sub pathRemote {return shift->{strPathRemote}};
sub storageLocal {return shift->{oStorageLocal}};
# sub storageEncrypt {return shift->{oStorageEncrypt}};
# sub storageRemote {return shift->{oStorageRemote}};
1;

36
test/patch/debian-package.patch Normal file
View File

@ -0,0 +1,36 @@
--- control
+++ control
@@ -4,7 +4,7 @@
Maintainer: Debian PostgreSQL Maintainers <team+postgresql@tracker.debian.org>
Uploaders: Adrian Vondendriesch <adrian.vondendriesch@credativ.de>
Build-Depends: debhelper (>= 9),
- libperl-dev,
+ perl,
libpq-dev,
libssl-dev,
libxml-checker-perl,
@@ -18,10 +18,8 @@
Package: pgbackrest
Architecture: any
-Depends: perl,
- postgresql-common,
+Depends: postgresql-common,
${misc:Depends},
- ${perl:Depends},
${shlibs:Depends}
Suggests: pgbackrest-doc
Description: Reliable PostgreSQL Backup & Restore
--- copyright
+++ copyright
@@ -3,8 +3,8 @@
Source: https://github.com/pgbackrest/pgbackrest
Files: *
-Copyright: 2015-2016 The PostgreSQL Global Development Group
- 2013-2016 David Steele
+Copyright: 2015-2020 The PostgreSQL Global Development Group
+ 2013-2020 David Steele
License: MIT
Files: debian/*

20
test/patch/rhel-package.patch Normal file
View File

@ -0,0 +1,20 @@
--- pgbackrest.spec
+++ pgbackrest.spec
@@ -8,16 +8,7 @@
Url: http://www.pgbackrest.org/
Source0: https://github.com/pgbackrest/pgbackrest/archive/release/%{version}.tar.gz
Source1: pgbackrest-conf.patch
-Requires: perl-XML-LibXML perl-IO-Socket-SSL
-%if 0%{?rhel} && 0%{?rhel} <= 6
-Requires: perl-parent perl-JSON perl-Time-HiRes
-%else
-Requires: perl-JSON-PP
-%endif
-Requires: perl-Digest-SHA perl-DBD-Pg perl-Time-HiRes zlib
-Requires: perl(:MODULE_COMPAT_%(eval "`%{__perl} -V:version`"; echo $version))
-BuildRequires: openssl-devel zlib-devel perl-ExtUtils-Embed
-BuildRequires: postgresql%{pgmajorversion}-devel
+BuildRequires: openssl-devel zlib-devel postgresql%{pgmajorversion}-devel
Requires: postgresql-libs

13
test/src/module/common/exitTest.c
View File

@ -107,19 +107,6 @@ testRun(void)
}
TRY_END();
// -------------------------------------------------------------------------------------------------------------------------
TRY_BEGIN()
{
THROW(RuntimeError, PERL_EMBED_ERROR);
}
CATCH_ANY()
{
exitSafe(0, true, signalTypeNone);
harnessLogResult(
"P00 INFO: archive-push command end: aborted with exception [122]");
}
TRY_END();
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_INT(
exitSafe(errorTypeCode(&TermError), false, signalTypeNone), errorTypeCode(&TermError), "exit on term with no signal");

112
test/src/module/perl/configTest.c
View File

@ -1,112 +0,0 @@
/***********************************************************************************************************************************
Test Perl Exec
***********************************************************************************************************************************/
#include "config/config.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("perlOptionJson()"))
{
cfgInit();
TEST_RESULT_STR(strPtr(perlOptionJson()), "{}", "no options");
// -------------------------------------------------------------------------------------------------------------------------
cfgInit();
cfgOptionValidSet(cfgOptCompress, true);
cfgOptionSet(cfgOptCompress, cfgSourceParam, varNewBool(true));
cfgOptionValidSet(cfgOptConfig, true);
cfgOptionNegateSet(cfgOptConfig, true);
cfgOptionSet(cfgOptConfig, cfgSourceParam, NULL);
cfgOptionValidSet(cfgOptOnline, true);
cfgOptionNegateSet(cfgOptOnline, true);
cfgOptionSet(cfgOptOnline, cfgSourceParam, varNewBool(false));
cfgOptionValidSet(cfgOptPgHost, true);
cfgOptionResetSet(cfgOptPgHost, true);
cfgOptionValidSet(cfgOptPgPath, true);
cfgOptionSet(cfgOptPgPath, cfgSourceConfig, varNewStr(strNew("/path/db/pg")));
cfgOptionValidSet(cfgOptBackupStandby, true);
cfgOptionResetSet(cfgOptBackupStandby, true);
cfgOptionSet(cfgOptBackupStandby, cfgSourceParam, varNewBool(false));
cfgOptionValidSet(cfgOptProtocolTimeout, true);
cfgOptionSet(cfgOptProtocolTimeout, cfgSourceParam, varNewDbl(1.1));
cfgOptionValidSet(cfgOptArchivePushQueueMax, true);
cfgOptionSet(cfgOptArchivePushQueueMax, cfgSourceParam, varNewInt64(999999999999));
cfgOptionValidSet(cfgOptRepoCipherPass, true);
cfgOptionSet(cfgOptRepoCipherPass, cfgSourceConfig, varNewStr(strNew("part1\npart2")));
cfgOptionValidSet(cfgOptCompressLevel, true);
cfgOptionSet(cfgOptCompressLevel, cfgSourceConfig, varNewInt(3));
cfgOptionValidSet(cfgOptStanza, true);
cfgOptionSet(cfgOptStanza, cfgSourceDefault, varNewStr(strNew("db")));
TEST_RESULT_STR(
strPtr(perlOptionJson()),
"{"
"\"archive-push-queue-max\":"
"{\"negate\":false,\"reset\":false,\"source\":\"param\",\"valid\":true,\"value\":999999999999},"
"\"backup-standby\":{\"negate\":false,\"reset\":true,\"source\":\"param\",\"valid\":true,\"value\":false},"
"\"compress\":{\"negate\":false,\"reset\":false,\"source\":\"param\",\"valid\":true,\"value\":true},"
"\"compress-level\":{\"negate\":false,\"reset\":false,\"source\":\"config\",\"valid\":true,\"value\":3},"
"\"config\":{\"negate\":true,\"reset\":false,\"source\":\"param\",\"valid\":true},"
"\"online\":{\"negate\":true,\"reset\":false,\"source\":\"param\",\"valid\":true,\"value\":false},"
"\"pg1-host\":{\"negate\":false,\"reset\":true,\"source\":\"default\",\"valid\":true},"
"\"pg1-path\":{\"negate\":false,\"reset\":false,\"source\":\"config\",\"valid\":true,\"value\":\"/path/db/pg\"},"
"\"protocol-timeout\":{\"negate\":false,\"reset\":false,\"source\":\"param\",\"valid\":true,\"value\":1.1},"
"\"repo1-cipher-pass\":{\"negate\":false,\"reset\":false,\"source\":\"config\",\"valid\":true,"
"\"value\":\"part1\\npart2\"},"
"\"stanza\":{\"negate\":false,\"reset\":false,\"source\":\"default\",\"valid\":true,\"value\":\"db\"}"
"}",
"simple options");
// -------------------------------------------------------------------------------------------------------------------------
cfgInit();
cfgOptionValidSet(cfgOptDbInclude, true);
StringList *includeList = strLstNew();
strLstAdd(includeList, strNew("db1"));
strLstAdd(includeList, strNew("db2"));
cfgOptionSet(cfgOptDbInclude, cfgSourceParam, varNewVarLst(varLstNewStrLst(includeList)));
cfgOptionValidSet(cfgOptRecoveryOption, true);
Variant *recoveryVar = varNewKv(kvNew());
KeyValue *recoveryKv = varKv(recoveryVar);
kvPut(recoveryKv, varNewStr(strNew("standby_mode")), varNewStr(strNew("on")));
kvPut(recoveryKv, varNewStr(strNew("primary_conn_info")), varNewStr(strNew("blah")));
cfgOptionSet(cfgOptRecoveryOption, cfgSourceParam, recoveryVar);
StringList *commandParamList = strLstNew();
strLstAdd(commandParamList, strNew("param1"));
strLstAdd(commandParamList, strNew("param2"));
cfgCommandParamSet(commandParamList);
TEST_RESULT_STR(
strPtr(perlOptionJson()),
"{"
"\"db-include\":{\"negate\":false,\"reset\":false,\"source\":\"param\",\"valid\":true,"
"\"value\":{\"db1\":true,\"db2\":true}},"
"\"recovery-option\":{\"negate\":false,\"reset\":false,\"source\":\"param\",\"valid\":true,"
"\"value\":{\"primary_conn_info\":\"blah\",\"standby_mode\":\"on\"}}"
"}",
"complex options");
}
FUNCTION_HARNESS_RESULT_VOID();
}

82
test/src/module/perl/execTest.c
View File

@ -1,82 +0,0 @@
/***********************************************************************************************************************************
Test Perl Exec
***********************************************************************************************************************************/
#include "config/config.h"
#include "config/load.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("perlMain()"))
{
// -------------------------------------------------------------------------------------------------------------------------
cfgInit();
cfgCommandSet(cfgCmdInfo);
cfgExeSet(strNew("/path/to/pgbackrest"));
TEST_RESULT_STR(
strPtr(perlMain()), "($iResult, $bErrorC, $strMessage) = pgBackRest::Main::main('info')", "command with no options");
// -------------------------------------------------------------------------------------------------------------------------
cfgOptionValidSet(cfgOptCompress, true);
cfgOptionSet(cfgOptCompress, cfgSourceParam, varNewBool(true));
StringList *commandParamList = strLstNew();
strLstAdd(commandParamList, strNew("A"));
strLstAdd(commandParamList, strNew("B"));
cfgCommandParamSet(commandParamList);
TEST_RESULT_STR(
strPtr(perlMain()), "($iResult, $bErrorC, $strMessage) = pgBackRest::Main::main('info','A','B')",
"command with one option and params");
}
// *****************************************************************************************************************************
if (testBegin("embeddedModuleGetInternal()"))
{
TEST_ERROR(embeddedModuleGetInternal(BOGUS_STR), AssertError, "unable to load embedded module 'BOGUS'");
}
// *****************************************************************************************************************************
if (testBegin("perlExecResult()"))
{
TRY_BEGIN()
{
THROW(PathMissingError, "test message");
}
CATCH_ANY()
{
TEST_ERROR(perlExecResult(errorTypeCode(&PathMissingError), true, NULL), PathMissingError, "test message");
}
TRY_END();
TEST_RESULT_INT(perlExecResult(0, false, NULL), 0, "test success");
}
// *****************************************************************************************************************************
if (testBegin("perlInit(), perlExec(), and perlFree()"))
{
StringList *argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew("--log-level-console=off"));
strLstAdd(argList, strNew("--log-level-stderr=off"));
strLstAdd(argList, strNew("--log-level-file=off"));
strLstAdd(argList, strNew("archive-push"));
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "load archive-push config");
TEST_RESULT_VOID(perlFree(0), "free Perl before it is init'd");
TEST_RESULT_VOID(perlInit(), "init Perl");
TEST_ERROR(perlExec(), PathMissingError, PERL_EMBED_ERROR);
TEST_RESULT_VOID(perlFree(0), "free Perl");
}
FUNCTION_HARNESS_RESULT_VOID();
}

12
test/src/module/protocol/protocolTest.c
View File

@ -220,7 +220,7 @@ testRun(void)
strPtr(
strNew(
"-o|LogLevel=error|-o|Compression=no|-o|PasswordAuthentication=no|repo-host-user@repo-host"
"|pgbackrest --c --command=archive-get --log-level-file=off --log-level-stderr=error --process=0"
"|pgbackrest --command=archive-get --log-level-file=off --log-level-stderr=error --process=0"
" --stanza=test1 --type=backup remote")),
"remote protocol params");
@ -243,7 +243,7 @@ testRun(void)
strPtr(
strNew(
"-o|LogLevel=error|-o|Compression=no|-o|PasswordAuthentication=no|-p|444|repo-host-user@repo-host"
"|pgbackrest --c --command=archive-get --config=/path/pgbackrest.conf --config-include-path=/path/include"
"|pgbackrest --command=archive-get --config=/path/pgbackrest.conf --config-include-path=/path/include"
" --config-path=/path/config --log-level-file=info --log-level-stderr=error --log-subprocess --process=1"
" --stanza=test1 --type=backup remote")),
"remote protocol params with replacements");
@ -265,7 +265,7 @@ testRun(void)
strPtr(
strNew(
"-o|LogLevel=error|-o|Compression=no|-o|PasswordAuthentication=no|pgbackrest@repo-host"
"|pgbackrest --c --command=archive-get --log-level-file=off --log-level-stderr=error --process=3"
"|pgbackrest --command=archive-get --log-level-file=off --log-level-stderr=error --process=3"
" --stanza=test1 --type=backup remote")),
"remote protocol params for backup local");
@ -284,7 +284,7 @@ testRun(void)
strPtr(
strNew(
"-o|LogLevel=error|-o|Compression=no|-o|PasswordAuthentication=no|postgres@pg1-host"
"|pgbackrest --c --command=backup --log-level-file=off --log-level-stderr=error --pg1-path=/path/to/1"
"|pgbackrest --command=backup --log-level-file=off --log-level-stderr=error --pg1-path=/path/to/1"
" --process=1 --stanza=test1 --type=db remote")),
"remote protocol params for db backup");
@ -309,7 +309,7 @@ testRun(void)
strPtr(
strNew(
"-o|LogLevel=error|-o|Compression=no|-o|PasswordAuthentication=no|postgres@pg2-host"
"|pgbackrest --c --command=backup --log-level-file=off --log-level-stderr=error --pg1-path=/path/to/2"
"|pgbackrest --command=backup --log-level-file=off --log-level-stderr=error --pg1-path=/path/to/2"
" --process=4 --stanza=test1 --type=db remote")),
"remote protocol params for db local");
@ -334,7 +334,7 @@ testRun(void)
strPtr(
strNew(
"-o|LogLevel=error|-o|Compression=no|-o|PasswordAuthentication=no|postgres@pg3-host"
"|pgbackrest --c --command=backup --log-level-file=off --log-level-stderr=error --pg1-path=/path/to/3"
"|pgbackrest --command=backup --log-level-file=off --log-level-stderr=error --pg1-path=/path/to/3"
" --pg1-port=3333 --pg1-socket-path=/socket3 --process=4 --stanza=test1 --type=db remote")),
"remote protocol params for db local");
}

5
test/src/module/storage/posixTest.c
View File

@ -544,11 +544,6 @@ testRun(void)
storagePathP(storageTest, strNew("/path/toot")), AssertError,
"absolute path '/path/toot' is not in base path '/path/to'");
// Path enforcement disabled globally
storagePathEnforceSet(storageTest, false);
TEST_RESULT_STR(strPtr(storagePathP(storageTest, strNew("/bogus"))), "/bogus", "path enforce disabled globally");
storagePathEnforceSet(storageTest, true);
// Path enforcement disabled for a single call
TEST_RESULT_STR(strPtr(storagePathP(storageTest, strNew("/bogus"), .noEnforce = true)), "/bogus", "path enforce disabled");

56
test/test.pl
View File

@ -38,7 +38,6 @@ use pgBackRestBuild::Build::Common;
use pgBackRestBuild::Config::Build;
use pgBackRestBuild::Config::BuildDefine;
use pgBackRestBuild::Config::BuildParse;
use pgBackRestBuild::Embed::Build;
use pgBackRestBuild::Error::Build;
use pgBackRestBuild::Error::Data;
@ -567,59 +566,6 @@ eval
}
}
# Auto-generate C library code to embed in the binary
#-----------------------------------------------------------------------------------------------------------------------
if ($bGenLibC && (!$bSmart || grep(/^libc\//, @stryModifiedList)))
{
my $strLibC = executeTest(
"cd ${strBackRestBase}/libc && " .
"perl /usr/share/perl/5.26/ExtUtils/xsubpp -typemap /usr/share/perl/5.26/ExtUtils/typemap" .
" -typemap typemap LibC.xs");
# Trim off any trailing LFs
$strLibC = trim($strLibC) . "\n";
# Strip out line numbers. These are useful for the LibC build but only cause churn in the binary
# build.
$strLibC =~ s/^\#line .*\n//mg;
# Save into the bin src dir
my @stryBuilt;
my $strBuilt = 'src/perl/libc.auto.c';
if (buildPutDiffers($oStorageBackRest, "${strBackRestBase}/${strBuilt}", $strLibC))
{
push(@stryBuilt, $strBuilt);
push(@stryBuiltAll, @stryBuilt);
push(@stryModifiedList, @stryBuilt);
}
&log(INFO, " autogenerated embedded C code: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes'));
}
# Auto-generate embedded Perl code
#-----------------------------------------------------------------------------------------------------------------------
if (!$bSmart || grep(/^lib\//, @stryModifiedList))
{
my $rhBuild =
{
'embed' =>
{
&BLD_DATA => buildEmbed($oStorageBackRest),
&BLD_PATH => 'perl',
},
};
my @stryBuilt = buildAll("${strBackRestBase}/src", $rhBuild);
&log(INFO, " autogenerated embedded Perl code: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes'));
if (@stryBuilt)
{
push(@stryBuiltAll, @stryBuilt);
push(@stryModifiedList, @stryBuilt);
}
}
# Auto-generate C Makefile
#-----------------------------------------------------------------------------------------------------------------------
if (!$bSmart || grep(/^(src|libc)\//, @stryModifiedList))
@ -778,7 +724,7 @@ eval
foreach my $hTest (@{$oyTestRun})
{
# Bin build required for all Perl tests or if a C unit test calls Perl
if (!$hTest->{&TEST_C} || $hTest->{&TEST_PERL_REQ})
if (!$hTest->{&TEST_C} || $hTest->{&TEST_BIN_REQ})
{
$bBinRequired = true;
}