db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-04-15 11:36:40 +02:00
Code Issues Releases Activity

Remove LibC.

Browse Source 
This was the interface between Perl and C introduced in 36a5349b but since f0ef73db has only been used by the Perl integration tests.  This is expensive code to maintain just for testing.

The main dependency was the interface to storage, no matter where it was located, e.g. S3.  Replace this with the new-introduced repo commands (d3c83453) that allow access to repo storage via the command line.

The other dependency was on various cfgOption* functions and CFGOPT_ constants that were convenient but not necessary.  Replace these with hard-coded strings in most places and create new constants for commonly used values.

Remove all auto-generated Perl code.  This means that the error list will no longer be maintained automatically so copy used errors to Common::Exception.pm.  This file will need to be maintained manually going forward but there is not likely to be much churn as the Perl integration tests are being retired.

Update test.pl and related code to remove LibC builds.

Ding, dong, LibC is dead.
This commit is contained in:
David Steele 2020-03-09 17:41:59 -04:00
parent d3c83453de
commit 79cfd3aebf
62 changed files with 1203 additions and 4643 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/xml/user-guide.xml
View File

@ -730,9 +730,6 @@
<execute user="root" show="n">
<exe-cmd>mkdir -p /build/pgbackrest-release-{[version]}</exe-cmd>
</execute>
<execute user="root" show="n">
<exe-cmd>cp -r {[pgbackrest-repo-path]}/libc /build/pgbackrest-release-{[version]}</exe-cmd>
</execute>
<execute user="root" show="n">
<exe-cmd>cp -r {[pgbackrest-repo-path]}/src /build/pgbackrest-release-{[version]}</exe-cmd>
</execute>

2
lib/pgBackRest/Archive/Common.pm
View File

@ -17,8 +17,6 @@ use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
####################################################################################################################################

8
lib/pgBackRest/Archive/Info.pm
View File

@ -19,13 +19,11 @@ use File::Basename qw(dirname basename);
use pgBackRest::Archive::Common;
use pgBackRest::Common::Exception;
use pgBackRest::Config::Config;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::DbVersion;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
@ -95,9 +93,9 @@ sub new
# Init object and store variables
eval
{
$self = $class->SUPER::new($strArchiveInfoFile, {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing,
oStorage => storageRepo(), strCipherPass => storageRepo()->cipherPassUser(),
strCipherPassSub => $strCipherPassSub});
$self = $class->SUPER::new(
storageRepo(), $strArchiveInfoFile,
{bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub});
return true;
}
or do

19
lib/pgBackRest/Backup/Common.pm
View File

@ -14,10 +14,8 @@ use File::Basename;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Manifest;
use pgBackRest::Storage::Helper;
####################################################################################################################################
# Latest backup link constant
@ -25,6 +23,13 @@ use pgBackRest::Manifest;
use constant LINK_LATEST => 'latest';
push @EXPORT, qw(LINK_LATEST);
use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL);
use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF);
use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR);
####################################################################################################################################
# backupRegExpGet
#
@ -200,6 +205,7 @@ sub backupLabel
(
$strOperation,
$oStorageRepo,
$strRepoBackupPath,
$strType,
$strBackupLabelLast,
$lTimestampStart
@ -208,6 +214,7 @@ sub backupLabel
(
__PACKAGE__ . '::backupLabelFormat', \@_,
{name => 'oStorageRepo', trace => true},
{name => 'strRepoBackupPath', trace => true},
{name => 'strType', trace => true},
{name => 'strBackupLabelLast', required => false, trace => true},
{name => 'lTimestampStart', trace => true}
@ -221,15 +228,15 @@ sub backupLabel
# clocks. In practice this is most useful for making offline testing faster since it allows the wait after manifest build to
# be skipped by dealing with any backup label collisions here.
if ($oStorageRepo->list(
STORAGE_REPO_BACKUP,
$strRepoBackupPath,
{strExpression =>
($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStart) .
($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? 'F' : '(D|I)$')}) ||
$oStorageRepo->list(
STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTimestampStart),
"${strRepoBackupPath}/" . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTimestampStart),
{strExpression =>
($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStart) .
($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? 'F' : '(D|I)\.manifest\.' . COMPRESS_EXT . qw{$}),
($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? 'F' : '(D|I)\.manifest\.gz$'),
bIgnoreMissing => true}))
{
waitRemainder();

8
lib/pgBackRest/Backup/Info.pm
View File

@ -19,10 +19,8 @@ use pgBackRest::Backup::Common;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
####################################################################################################################################
@ -137,9 +135,9 @@ sub new
# Init object and store variables
eval
{
$self = $class->SUPER::new($strBackupInfoFile, {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing,
oStorage => $oStorage, strCipherPass => $oStorage->cipherPassUser(),
strCipherPassSub => $strCipherPassSub});
$self = $class->SUPER::new(
$oStorage, $strBackupInfoFile,
{bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub});
return true;
}
or do

96
lib/pgBackRest/Common/Exception.pm
View File

@ -12,12 +12,98 @@ use Scalar::Util qw(blessed);
use Exporter qw(import);
our @EXPORT = qw();
use pgBackRest::Common::ExceptionAuto;
####################################################################################################################################
# Error Definitions
####################################################################################################################################
use constant ERROR_MINIMUM => 25;
push @EXPORT, qw(ERROR_MINIMUM);
use constant ERROR_MAXIMUM => 125;
push @EXPORT, qw(ERROR_MAXIMUM);
####################################################################################################################################
# Export error constants
####################################################################################################################################
push(@EXPORT, @pgBackRest::Common::ExceptionAuto::EXPORT);
use constant ERROR_ASSERT => 25;
push @EXPORT, qw(ERROR_ASSERT);
use constant ERROR_CHECKSUM => 26;
push @EXPORT, qw(ERROR_CHECKSUM);
use constant ERROR_CONFIG => 27;
push @EXPORT, qw(ERROR_CONFIG);
use constant ERROR_FILE_INVALID => 28;
push @EXPORT, qw(ERROR_FILE_INVALID);
use constant ERROR_FORMAT => 29;
push @EXPORT, qw(ERROR_FORMAT);
use constant ERROR_OPTION_INVALID_VALUE => 32;
push @EXPORT, qw(ERROR_OPTION_INVALID_VALUE);
use constant ERROR_POSTMASTER_RUNNING => 38;
push @EXPORT, qw(ERROR_POSTMASTER_RUNNING);
use constant ERROR_PATH_NOT_EMPTY => 40;
push @EXPORT, qw(ERROR_PATH_NOT_EMPTY);
use constant ERROR_FILE_OPEN => 41;
push @EXPORT, qw(ERROR_FILE_OPEN);
use constant ERROR_FILE_READ => 42;
push @EXPORT, qw(ERROR_FILE_READ);
use constant ERROR_ARCHIVE_MISMATCH => 44;
push @EXPORT, qw(ERROR_ARCHIVE_MISMATCH);
use constant ERROR_ARCHIVE_DUPLICATE => 45;
push @EXPORT, qw(ERROR_ARCHIVE_DUPLICATE);
use constant ERROR_PATH_CREATE => 47;
push @EXPORT, qw(ERROR_PATH_CREATE);
use constant ERROR_LOCK_ACQUIRE => 50;
push @EXPORT, qw(ERROR_LOCK_ACQUIRE);
use constant ERROR_BACKUP_MISMATCH => 51;
push @EXPORT, qw(ERROR_BACKUP_MISMATCH);
use constant ERROR_PATH_OPEN => 53;
push @EXPORT, qw(ERROR_PATH_OPEN);
use constant ERROR_PATH_SYNC => 54;
push @EXPORT, qw(ERROR_PATH_SYNC);
use constant ERROR_FILE_MISSING => 55;
push @EXPORT, qw(ERROR_FILE_MISSING);
use constant ERROR_DB_CONNECT => 56;
push @EXPORT, qw(ERROR_DB_CONNECT);
use constant ERROR_DB_QUERY => 57;
push @EXPORT, qw(ERROR_DB_QUERY);
use constant ERROR_DB_MISMATCH => 58;
push @EXPORT, qw(ERROR_DB_MISMATCH);
use constant ERROR_PATH_REMOVE => 61;
push @EXPORT, qw(ERROR_PATH_REMOVE);
use constant ERROR_STOP => 62;
push @EXPORT, qw(ERROR_STOP);
use constant ERROR_FILE_WRITE => 64;
push @EXPORT, qw(ERROR_FILE_WRITE);
use constant ERROR_FEATURE_NOT_SUPPORTED => 67;
push @EXPORT, qw(ERROR_FEATURE_NOT_SUPPORTED);
use constant ERROR_ARCHIVE_COMMAND_INVALID => 68;
push @EXPORT, qw(ERROR_ARCHIVE_COMMAND_INVALID);
use constant ERROR_LINK_EXPECTED => 69;
push @EXPORT, qw(ERROR_LINK_EXPECTED);
use constant ERROR_LINK_DESTINATION => 70;
push @EXPORT, qw(ERROR_LINK_DESTINATION);
use constant ERROR_PATH_MISSING => 73;
push @EXPORT, qw(ERROR_PATH_MISSING);
use constant ERROR_FILE_MOVE => 74;
push @EXPORT, qw(ERROR_FILE_MOVE);
use constant ERROR_PATH_TYPE => 77;
push @EXPORT, qw(ERROR_PATH_TYPE);
use constant ERROR_DB_MISSING => 80;
push @EXPORT, qw(ERROR_DB_MISSING);
use constant ERROR_DB_INVALID => 81;
push @EXPORT, qw(ERROR_DB_INVALID);
use constant ERROR_ARCHIVE_TIMEOUT => 82;
push @EXPORT, qw(ERROR_ARCHIVE_TIMEOUT);
use constant ERROR_ARCHIVE_DISABLED => 87;
push @EXPORT, qw(ERROR_ARCHIVE_DISABLED);
use constant ERROR_FILE_OWNER => 88;
push @EXPORT, qw(ERROR_FILE_OWNER);
use constant ERROR_PATH_EXISTS => 92;
push @EXPORT, qw(ERROR_PATH_EXISTS);
use constant ERROR_FILE_EXISTS => 93;
push @EXPORT, qw(ERROR_FILE_EXISTS);
use constant ERROR_CRYPTO => 95;
push @EXPORT, qw(ERROR_CRYPTO);
use constant ERROR_INVALID => 123;
push @EXPORT, qw(ERROR_INVALID);
use constant ERROR_UNHANDLED => 124;
push @EXPORT, qw(ERROR_UNHANDLED);
use constant ERROR_UNKNOWN => 125;
push @EXPORT, qw(ERROR_UNKNOWN);
####################################################################################################################################
# CONSTRUCTOR

183
lib/pgBackRest/Common/ExceptionAuto.pm
View File

@ -1,183 +0,0 @@
####################################################################################################################################
# COMMON EXCEPTION AUTO MODULE
#
# Automatically generated by Build.pm -- do not modify directly.
####################################################################################################################################
package pgBackRest::Common::ExceptionAuto;
use strict;
use warnings FATAL => qw(all);
use Exporter qw(import);
our @EXPORT = qw();
####################################################################################################################################
# Error Definitions
####################################################################################################################################
use constant ERROR_MINIMUM => 25;
push @EXPORT, qw(ERROR_MINIMUM);
use constant ERROR_MAXIMUM => 125;
push @EXPORT, qw(ERROR_MAXIMUM);
use constant ERROR_ASSERT => 25;
push @EXPORT, qw(ERROR_ASSERT);
use constant ERROR_CHECKSUM => 26;
push @EXPORT, qw(ERROR_CHECKSUM);
use constant ERROR_CONFIG => 27;
push @EXPORT, qw(ERROR_CONFIG);
use constant ERROR_FILE_INVALID => 28;
push @EXPORT, qw(ERROR_FILE_INVALID);
use constant ERROR_FORMAT => 29;
push @EXPORT, qw(ERROR_FORMAT);
use constant ERROR_COMMAND_REQUIRED => 30;
push @EXPORT, qw(ERROR_COMMAND_REQUIRED);
use constant ERROR_OPTION_INVALID => 31;
push @EXPORT, qw(ERROR_OPTION_INVALID);
use constant ERROR_OPTION_INVALID_VALUE => 32;
push @EXPORT, qw(ERROR_OPTION_INVALID_VALUE);
use constant ERROR_OPTION_INVALID_RANGE => 33;
push @EXPORT, qw(ERROR_OPTION_INVALID_RANGE);
use constant ERROR_OPTION_INVALID_PAIR => 34;
push @EXPORT, qw(ERROR_OPTION_INVALID_PAIR);
use constant ERROR_OPTION_DUPLICATE_KEY => 35;
push @EXPORT, qw(ERROR_OPTION_DUPLICATE_KEY);
use constant ERROR_OPTION_NEGATE => 36;
push @EXPORT, qw(ERROR_OPTION_NEGATE);
use constant ERROR_OPTION_REQUIRED => 37;
push @EXPORT, qw(ERROR_OPTION_REQUIRED);
use constant ERROR_POSTMASTER_RUNNING => 38;
push @EXPORT, qw(ERROR_POSTMASTER_RUNNING);
use constant ERROR_PROTOCOL => 39;
push @EXPORT, qw(ERROR_PROTOCOL);
use constant ERROR_PATH_NOT_EMPTY => 40;
push @EXPORT, qw(ERROR_PATH_NOT_EMPTY);
use constant ERROR_FILE_OPEN => 41;
push @EXPORT, qw(ERROR_FILE_OPEN);
use constant ERROR_FILE_READ => 42;
push @EXPORT, qw(ERROR_FILE_READ);
use constant ERROR_PARAM_REQUIRED => 43;
push @EXPORT, qw(ERROR_PARAM_REQUIRED);
use constant ERROR_ARCHIVE_MISMATCH => 44;
push @EXPORT, qw(ERROR_ARCHIVE_MISMATCH);
use constant ERROR_ARCHIVE_DUPLICATE => 45;
push @EXPORT, qw(ERROR_ARCHIVE_DUPLICATE);
use constant ERROR_VERSION_NOT_SUPPORTED => 46;
push @EXPORT, qw(ERROR_VERSION_NOT_SUPPORTED);
use constant ERROR_PATH_CREATE => 47;
push @EXPORT, qw(ERROR_PATH_CREATE);
use constant ERROR_COMMAND_INVALID => 48;
push @EXPORT, qw(ERROR_COMMAND_INVALID);
use constant ERROR_HOST_CONNECT => 49;
push @EXPORT, qw(ERROR_HOST_CONNECT);
use constant ERROR_LOCK_ACQUIRE => 50;
push @EXPORT, qw(ERROR_LOCK_ACQUIRE);
use constant ERROR_BACKUP_MISMATCH => 51;
push @EXPORT, qw(ERROR_BACKUP_MISMATCH);
use constant ERROR_FILE_SYNC => 52;
push @EXPORT, qw(ERROR_FILE_SYNC);
use constant ERROR_PATH_OPEN => 53;
push @EXPORT, qw(ERROR_PATH_OPEN);
use constant ERROR_PATH_SYNC => 54;
push @EXPORT, qw(ERROR_PATH_SYNC);
use constant ERROR_FILE_MISSING => 55;
push @EXPORT, qw(ERROR_FILE_MISSING);
use constant ERROR_DB_CONNECT => 56;
push @EXPORT, qw(ERROR_DB_CONNECT);
use constant ERROR_DB_QUERY => 57;
push @EXPORT, qw(ERROR_DB_QUERY);
use constant ERROR_DB_MISMATCH => 58;
push @EXPORT, qw(ERROR_DB_MISMATCH);
use constant ERROR_DB_TIMEOUT => 59;
push @EXPORT, qw(ERROR_DB_TIMEOUT);
use constant ERROR_FILE_REMOVE => 60;
push @EXPORT, qw(ERROR_FILE_REMOVE);
use constant ERROR_PATH_REMOVE => 61;
push @EXPORT, qw(ERROR_PATH_REMOVE);
use constant ERROR_STOP => 62;
push @EXPORT, qw(ERROR_STOP);
use constant ERROR_TERM => 63;
push @EXPORT, qw(ERROR_TERM);
use constant ERROR_FILE_WRITE => 64;
push @EXPORT, qw(ERROR_FILE_WRITE);
use constant ERROR_PROTOCOL_TIMEOUT => 66;
push @EXPORT, qw(ERROR_PROTOCOL_TIMEOUT);
use constant ERROR_FEATURE_NOT_SUPPORTED => 67;
push @EXPORT, qw(ERROR_FEATURE_NOT_SUPPORTED);
use constant ERROR_ARCHIVE_COMMAND_INVALID => 68;
push @EXPORT, qw(ERROR_ARCHIVE_COMMAND_INVALID);
use constant ERROR_LINK_EXPECTED => 69;
push @EXPORT, qw(ERROR_LINK_EXPECTED);
use constant ERROR_LINK_DESTINATION => 70;
push @EXPORT, qw(ERROR_LINK_DESTINATION);
use constant ERROR_HOST_INVALID => 72;
push @EXPORT, qw(ERROR_HOST_INVALID);
use constant ERROR_PATH_MISSING => 73;
push @EXPORT, qw(ERROR_PATH_MISSING);
use constant ERROR_FILE_MOVE => 74;
push @EXPORT, qw(ERROR_FILE_MOVE);
use constant ERROR_BACKUP_SET_INVALID => 75;
push @EXPORT, qw(ERROR_BACKUP_SET_INVALID);
use constant ERROR_TABLESPACE_MAP => 76;
push @EXPORT, qw(ERROR_TABLESPACE_MAP);
use constant ERROR_PATH_TYPE => 77;
push @EXPORT, qw(ERROR_PATH_TYPE);
use constant ERROR_LINK_MAP => 78;
push @EXPORT, qw(ERROR_LINK_MAP);
use constant ERROR_FILE_CLOSE => 79;
push @EXPORT, qw(ERROR_FILE_CLOSE);
use constant ERROR_DB_MISSING => 80;
push @EXPORT, qw(ERROR_DB_MISSING);
use constant ERROR_DB_INVALID => 81;
push @EXPORT, qw(ERROR_DB_INVALID);
use constant ERROR_ARCHIVE_TIMEOUT => 82;
push @EXPORT, qw(ERROR_ARCHIVE_TIMEOUT);
use constant ERROR_FILE_MODE => 83;
push @EXPORT, qw(ERROR_FILE_MODE);
use constant ERROR_OPTION_MULTIPLE_VALUE => 84;
push @EXPORT, qw(ERROR_OPTION_MULTIPLE_VALUE);
use constant ERROR_PROTOCOL_OUTPUT_REQUIRED => 85;
push @EXPORT, qw(ERROR_PROTOCOL_OUTPUT_REQUIRED);
use constant ERROR_LINK_OPEN => 86;
push @EXPORT, qw(ERROR_LINK_OPEN);
use constant ERROR_ARCHIVE_DISABLED => 87;
push @EXPORT, qw(ERROR_ARCHIVE_DISABLED);
use constant ERROR_FILE_OWNER => 88;
push @EXPORT, qw(ERROR_FILE_OWNER);
use constant ERROR_USER_MISSING => 89;
push @EXPORT, qw(ERROR_USER_MISSING);
use constant ERROR_OPTION_COMMAND => 90;
push @EXPORT, qw(ERROR_OPTION_COMMAND);
use constant ERROR_GROUP_MISSING => 91;
push @EXPORT, qw(ERROR_GROUP_MISSING);
use constant ERROR_PATH_EXISTS => 92;
push @EXPORT, qw(ERROR_PATH_EXISTS);
use constant ERROR_FILE_EXISTS => 93;
push @EXPORT, qw(ERROR_FILE_EXISTS);
use constant ERROR_MEMORY => 94;
push @EXPORT, qw(ERROR_MEMORY);
use constant ERROR_CRYPTO => 95;
push @EXPORT, qw(ERROR_CRYPTO);
use constant ERROR_PARAM_INVALID => 96;
push @EXPORT, qw(ERROR_PARAM_INVALID);
use constant ERROR_PATH_CLOSE => 97;
push @EXPORT, qw(ERROR_PATH_CLOSE);
use constant ERROR_FILE_INFO => 98;
push @EXPORT, qw(ERROR_FILE_INFO);
use constant ERROR_JSON_FORMAT => 99;
push @EXPORT, qw(ERROR_JSON_FORMAT);
use constant ERROR_KERNEL => 100;
push @EXPORT, qw(ERROR_KERNEL);
use constant ERROR_SERVICE => 101;
push @EXPORT, qw(ERROR_SERVICE);
use constant ERROR_EXECUTE => 102;
push @EXPORT, qw(ERROR_EXECUTE);
use constant ERROR_RUNTIME => 122;
push @EXPORT, qw(ERROR_RUNTIME);
use constant ERROR_INVALID => 123;
push @EXPORT, qw(ERROR_INVALID);
use constant ERROR_UNHANDLED => 124;
push @EXPORT, qw(ERROR_UNHANDLED);
use constant ERROR_UNKNOWN => 125;
push @EXPORT, qw(ERROR_UNKNOWN);
1;

18
lib/pgBackRest/Common/Ini.pm
View File

@ -74,17 +74,13 @@ sub new
my $self = {};
bless $self, $class;
# Load Storage::Helper module
require pgBackRest::Storage::Helper;
pgBackRest::Storage::Helper->import();
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{oStorage},
$self->{strFileName},
my $bLoad,
my $strContent,
$self->{oStorage},
$self->{iInitFormat},
$self->{strInitVersion},
my $bIgnoreMissing,
@ -94,10 +90,10 @@ sub new
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oStorage', trace => true},
{name => 'strFileName', trace => true},
{name => 'bLoad', optional => true, default => true, trace => true},
{name => 'strContent', optional => true, trace => true},
{name => 'oStorage', optional => true, default => storageLocal(), trace => true},
{name => 'iInitFormat', optional => true, default => REPOSITORY_FORMAT, trace => true},
{name => 'strInitVersion', optional => true, default => PROJECT_VERSION, trace => true},
{name => 'bIgnoreMissing', optional => true, default => false, trace => true},
@ -105,11 +101,6 @@ sub new
{name => 'strCipherPassSub', optional => true, trace => true},
);
if (defined($self->{oStorage}->cipherPassUser()) && !defined($self->{strCipherPass}))
{
confess &log(ERROR, 'passphrase is required when storage is encrypted', ERROR_CRYPTO);
}
# Set changed to false
$self->{bModified} = false;
@ -139,11 +130,6 @@ sub new
{
$self->set(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS, undef, $strCipherPassSub);
}
elsif ((defined($self->{strCipherPass}) && !defined($strCipherPassSub)) ||
(!defined($self->{strCipherPass}) && defined($strCipherPassSub)))
{
confess &log(ASSERT, 'a user passphrase and sub passphrase are both required when encrypting');
}
}
return $self;

192
lib/pgBackRest/Common/Io/Process.pm Normal file
View File

@ -0,0 +1,192 @@
####################################################################################################################################
# Process Execution, Management, and IO
####################################################################################################################################
package pgBackRest::Common::Io::Process;
use parent 'pgBackRest::Common::Io::Filter';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use IPC::Open3 qw(open3);
use POSIX qw(:sys_wait_h);
use Symbol 'gensym';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
####################################################################################################################################
# Amount of time to attempt to retrieve errors when a process terminates unexpectedly
####################################################################################################################################
use constant IO_ERROR_TIMEOUT => 5;
####################################################################################################################################
# new - use open3 to run the command and get the io handles
####################################################################################################################################
sub new
{
my $class = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oParent,
$strCommand,
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oParent', trace => true},
{name => 'strCommand', trace => true},
);
# Bless with new class
my $self = $class->SUPER::new($oParent);
bless $self, $class;
# Use open3 to run the command
my ($iProcessId, $fhRead, $fhWrite, $fhReadError);
$fhReadError = gensym;
$iProcessId = IPC::Open3::open3($fhWrite, $fhRead, $fhReadError, $strCommand);
# Set handles
$self->handleReadSet($fhRead);
$self->handleWriteSet($fhWrite);
# Set variables
$self->{iProcessId} = $iProcessId;
$self->{fhReadError} = $fhReadError;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# error - handle errors
####################################################################################################################################
sub error
{
my $self = shift;
my $iCode = shift;
my $strMessage = shift;
my $strDetail = shift;
my $bClose = shift;
if (defined($self->{iProcessId}))
{
my $oWait = waitInit(defined($iCode) ? IO_ERROR_TIMEOUT : 0);
do
{
# Check the result
my $iResult = waitpid($self->{iProcessId}, $bClose ? 0 : WNOHANG);
# Error if the process exited unexpectedly
if ($iResult != 0)
{
# Get the exit status
$self->{iExitStatus} = $iResult == -1 ? 255 : ${^CHILD_ERROR_NATIVE} >> 8;
# Drain the stderr stream
my $strError;
my $oIoError = new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle($self->id(), $self->{fhReadError}), 5, $self->bufferMax());
while (defined(my $strLine = $oIoError->readLine(true, false)))
{
$strError .= (defined($strError) ? "\n" : '') . $strLine;
}
delete($self->{iProcessId});
if ((!$bClose && $self->{iExitStatus} != 0) || defined($strError))
{
my $iErrorCode =
$self->{iExitStatus} >= ERROR_MINIMUM && $self->{iExitStatus} <= ERROR_MAXIMUM ?
$self->{iExitStatus} : ERROR_FILE_READ;
logErrorResult(
$iErrorCode, $self->id() . ' terminated unexpectedly' .
($self->{iExitStatus} != 255 ? sprintf(' [%03d]', $self->{iExitStatus}) : ''),
$strError);
}
}
}
while (waitMore($oWait));
if (defined($iCode))
{
$self->parent()->error($iCode, $strMessage, $strDetail);
}
}
else
{
confess &log(ASSERT, 'cannot call error() after process has been closed');
}
}
####################################################################################################################################
# Get process id
####################################################################################################################################
sub processId
{
my $self = shift;
return $self->{iProcessId};
}
####################################################################################################################################
# Get exit status (after close() is called)
####################################################################################################################################
sub exitStatus
{
my $self = shift;
return $self->{iExitStatus};
}
####################################################################################################################################
# writeLine - check for error before writing line
####################################################################################################################################
sub writeLine
{
my $self = shift;
my $strBuffer = shift;
# Check if the process has exited abnormally (doesn't seem like we should need this, but the next syswrite does a hard
# abort if the remote process has already closed)
$self->error();
return $self->parent()->writeLine($strBuffer);
}
####################################################################################################################################
# close - check if the process terminated on error
####################################################################################################################################
sub close
{
my $self = shift;
if (defined($self->{iProcessId}))
{
$self->error(undef, undef, undef, true);
# Class parent close
$self->parent()->close();
}
return true;
}
1;

453
lib/pgBackRest/Config/Config.pm
View File

@ -1,453 +0,0 @@
####################################################################################################################################
# CONFIG MODULE
####################################################################################################################################
package pgBackRest::Config::Config;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Io::Base;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::LibC qw(:config :configDefine);
use pgBackRest::Version;
####################################################################################################################################
# Export config constants and functions
####################################################################################################################################
push(@EXPORT, @{$pgBackRest::LibC::EXPORT_TAGS{config}});
push(@EXPORT, @{$pgBackRest::LibC::EXPORT_TAGS{configDefine}});
####################################################################################################################################
# SOURCE Constants
####################################################################################################################################
use constant CFGDEF_SOURCE_CONFIG => 'config';
push @EXPORT, qw(CFGDEF_SOURCE_CONFIG);
use constant CFGDEF_SOURCE_PARAM => 'param';
push @EXPORT, qw(CFGDEF_SOURCE_PARAM);
use constant CFGDEF_SOURCE_DEFAULT => 'default';
push @EXPORT, qw(CFGDEF_SOURCE_DEFAULT);
####################################################################################################################################
# Configuration section constants
####################################################################################################################################
use constant CFGDEF_SECTION_GLOBAL => 'global';
push @EXPORT, qw(CFGDEF_SECTION_GLOBAL);
use constant CFGDEF_SECTION_STANZA => 'stanza';
push @EXPORT, qw(CFGDEF_SECTION_STANZA);
####################################################################################################################################
# Module variables
####################################################################################################################################
my %oOption; # Option hash
my $strCommand; # Command (backup, archive-get, ...)
my $bInitLog = false; # Has logging been initialized yet?
####################################################################################################################################
# configLogging - configure logging based on options
####################################################################################################################################
sub configLogging
{
my $bLogInitForce = shift;
if ($bInitLog || (defined($bLogInitForce) && $bLogInitForce))
{
logLevelSet(
cfgOptionValid(CFGOPT_LOG_LEVEL_FILE) ? cfgOption(CFGOPT_LOG_LEVEL_FILE) : OFF,
cfgOptionValid(CFGOPT_LOG_LEVEL_CONSOLE) ? cfgOption(CFGOPT_LOG_LEVEL_CONSOLE) : OFF,
cfgOptionValid(CFGOPT_LOG_LEVEL_STDERR) ? cfgOption(CFGOPT_LOG_LEVEL_STDERR) : OFF,
cfgOptionValid(CFGOPT_LOG_TIMESTAMP) ? cfgOption(CFGOPT_LOG_TIMESTAMP) : undef,
cfgOptionValid(CFGOPT_PROCESS_MAX) ? cfgOption(CFGOPT_PROCESS_MAX) : undef);
$bInitLog = true;
}
}
push @EXPORT, qw(configLogging);
####################################################################################################################################
# Load configuration
#
# Additional conditions that cannot be codified by the option definitions are also tested here.
####################################################################################################################################
sub configLoad
{
my $bInitLogging = shift;
my $strBackRestBin = shift;
my $strCommandName = shift;
my $rstrConfigJson = shift;
# Set backrest bin
projectBinSet($strBackRestBin);
# Set command
$strCommand = $strCommandName;
eval
{
%oOption = %{(JSON::PP->new()->allow_nonref())->decode($$rstrConfigJson)};
return true;
}
or do
{
confess &log(ASSERT, "unable to parse config JSON");
};
# Load options into final option hash
for (my $iOptionId = 0; $iOptionId < cfgOptionTotal(); $iOptionId++)
{
my $strOptionName = cfgOptionName($iOptionId);
# If option is defined it is valid
if (defined($oOption{$strOptionName}))
{
# Convert JSON bool to standard bool that Perl understands
if (cfgDefOptionType($iOptionId) eq CFGDEF_TYPE_BOOLEAN && defined($oOption{$strOptionName}{value}))
{
$oOption{$strOptionName}{value} = $oOption{$strOptionName}{value} eq INI_TRUE ? true : false;
}
}
# Else it is not valid
else
{
$oOption{$strOptionName}{valid} = false;
}
}
# If this is not the remote and logging is allowed (to not overwrite log levels for tests) then set the log level so that
# INFO/WARN messages can be displayed (the user may still disable them). This should be run before any WARN logging is
# generated.
if (!defined($bInitLogging) || $bInitLogging)
{
configLogging(true);
}
return true;
}
push @EXPORT, qw(configLoad);
####################################################################################################################################
# cfgOptionIdFromIndex - return name for options that can be indexed (e.g. pg1-host, pg2-host).
####################################################################################################################################
sub cfgOptionIdFromIndex
{
my $iOptionId = shift;
my $iIndex = shift;
# If the option doesn't have a prefix it can't be indexed
$iIndex = defined($iIndex) ? $iIndex : 1;
my $strPrefix = cfgDefOptionPrefix($iOptionId);
if (!defined($strPrefix))
{
if ($iIndex > 1)
{
confess &log(ASSERT, "'" . cfgOptionName($iOptionId) . "' option does not allow indexing");
}
return $iOptionId;
}
return cfgOptionId("${strPrefix}${iIndex}" . substr(cfgOptionName($iOptionId), index(cfgOptionName($iOptionId), '-')));
}
push @EXPORT, qw(cfgOptionIdFromIndex);
####################################################################################################################################
# cfgOptionSource - how was the option set?
####################################################################################################################################
sub cfgOptionSource
{
my $iOptionId = shift;
cfgOptionValid($iOptionId, true);
return $oOption{cfgOptionName($iOptionId)}{source};
}
push @EXPORT, qw(cfgOptionSource);
####################################################################################################################################
# cfgOptionValid - is the option valid for the current command?
####################################################################################################################################
sub cfgOptionValid
{
my $iOptionId = shift;
my $bError = shift;
# If defined then this is the command help is being generated for so all valid checks should be against that command
my $iCommandId;
if (defined($strCommand))
{
$iCommandId = cfgCommandId($strCommand);
}
if (defined($iCommandId) && cfgDefOptionValid($iCommandId, $iOptionId))
{
return true;
}
if (defined($bError) && $bError)
{
my $strOption = cfgOptionName($iOptionId);
if (!defined($oOption{$strOption}))
{
confess &log(ASSERT, "option '${strOption}' does not exist");
}
confess &log(ASSERT, "option '${strOption}' not valid for command '" . cfgCommandName(cfgCommandGet()) . "'");
}
return false;
}
push @EXPORT, qw(cfgOptionValid);
####################################################################################################################################
# cfgOption - get option value
####################################################################################################################################
sub cfgOption
{
my $iOptionId = shift;
my $bRequired = shift;
cfgOptionValid($iOptionId, true);
my $strOption = cfgOptionName($iOptionId);
if (!defined($oOption{$strOption}{value}) && (!defined($bRequired) || $bRequired))
{
confess &log(ASSERT, "option ${strOption} is required");
}
return $oOption{$strOption}{value};
}
push @EXPORT, qw(cfgOption);
####################################################################################################################################
# cfgOptionDefault - get option default value
####################################################################################################################################
sub cfgOptionDefault
{
my $iOptionId = shift;
cfgOptionValid($iOptionId, true);
return cfgDefOptionDefault(cfgCommandId($strCommand), $iOptionId);
}
push @EXPORT, qw(cfgOptionDefault);
####################################################################################################################################
# cfgOptionSet - set option value and source
####################################################################################################################################
sub cfgOptionSet
{
my $iOptionId = shift;
my $oValue = shift;
my $bForce = shift;
my $strOption = cfgOptionName($iOptionId);
if (!cfgOptionValid($iOptionId, !defined($bForce) || !$bForce))
{
$oOption{$strOption}{valid} = true;
}
$oOption{$strOption}{source} = CFGDEF_SOURCE_PARAM;
$oOption{$strOption}{value} = $oValue;
}
push @EXPORT, qw(cfgOptionSet);
####################################################################################################################################
# cfgOptionTest - test if an option exists or has a specific value
####################################################################################################################################
sub cfgOptionTest
{
my $iOptionId = shift;
my $strValue = shift;
if (!cfgOptionValid($iOptionId))
{
return false;
}
if (defined($strValue))
{
return cfgOption($iOptionId) eq $strValue ? true : false;
}
return defined($oOption{cfgOptionName($iOptionId)}{value}) ? true : false;
}
push @EXPORT, qw(cfgOptionTest);
####################################################################################################################################
# cfgCommandGet - get the current command
####################################################################################################################################
sub cfgCommandGet
{
return cfgCommandId($strCommand);
}
push @EXPORT, qw(cfgCommandGet);
####################################################################################################################################
# cfgCommandTest - test that the current command is equal to the provided value
####################################################################################################################################
sub cfgCommandTest
{
my $iCommandIdTest = shift;
return cfgCommandName($iCommandIdTest) eq $strCommand;
}
push @EXPORT, qw(cfgCommandTest);
####################################################################################################################################
# Set current command
####################################################################################################################################
sub cfgCommandSet
{
my $iCommandId = shift;
$strCommand = cfgCommandName($iCommandId);
}
push @EXPORT, qw(cfgCommandSet);
####################################################################################################################################
# cfgCommandWrite - using the options for the current command, write the command string for another command
#
# For example, this can be used to write the archive-get command for recovery.conf during a restore.
####################################################################################################################################
sub cfgCommandWrite
{
my $iNewCommandId = shift;
my $bIncludeConfig = shift;
my $strExeString = shift;
my $bIncludeCommand = shift;
my $oOptionOverride = shift;
my $bDisplayOnly = shift;
# Set defaults
$strExeString = defined($strExeString) ? $strExeString : projectBin();
$bIncludeConfig = defined($bIncludeConfig) ? $bIncludeConfig : false;
$bIncludeCommand = defined($bIncludeCommand) ? $bIncludeCommand : true;
# Iterate the options to figure out which ones are not default and need to be written out to the new command string
for (my $iOptionId = 0; $iOptionId < cfgOptionTotal(); $iOptionId++)
{
my $strOption = cfgOptionName($iOptionId);
my $bSecure = cfgDefOptionSecure($iOptionId);
# Skip option if it is secure and should not be output in logs or the command line
next if (($bSecure || $iOptionId == CFGOPT_REPO_CIPHER_TYPE) && !$bDisplayOnly);
# Process any option id overrides first
if (defined($oOptionOverride->{$iOptionId}))
{
if (defined($oOptionOverride->{$iOptionId}{value}))
{
$strExeString .= cfgCommandWriteOptionFormat(
$strOption, false, $bSecure, {value => $oOptionOverride->{$iOptionId}{value}});
}
}
# And process overrides passed by string - this is used by Perl compatibility functions
elsif (defined($oOptionOverride->{$strOption}))
{
if (defined($oOptionOverride->{$strOption}{value}))
{
$strExeString .= cfgCommandWriteOptionFormat(
$strOption, false, $bSecure, {value => $oOptionOverride->{$strOption}{value}});
}
}
# else look for non-default options in the current configuration
elsif (cfgDefOptionValid($iNewCommandId, $iOptionId) &&
defined($oOption{$strOption}{value}) &&
($bIncludeConfig ?
$oOption{$strOption}{source} ne CFGDEF_SOURCE_DEFAULT : $oOption{$strOption}{source} eq CFGDEF_SOURCE_PARAM))
{
my $oValue;
my $bMulti = false;
# If this is a hash then it will break up into multiple command-line options
if (ref($oOption{$strOption}{value}) eq 'HASH')
{
$oValue = $oOption{$strOption}{value};
$bMulti = true;
}
# Else a single value but store it in a hash anyway to make processing below simpler
else
{
$oValue = {value => $oOption{$strOption}{value}};
}
$strExeString .= cfgCommandWriteOptionFormat($strOption, $bMulti, $bSecure, $oValue);
}
# Else is reset
elsif (cfgDefOptionValid($iNewCommandId, $iOptionId) && $oOption{$strOption}{reset})
{
$strExeString .= " --reset-${strOption}";
}
}
if ($bIncludeCommand)
{
$strExeString .= ' ' . cfgCommandName($iNewCommandId);
}
return $strExeString;
}
push @EXPORT, qw(cfgCommandWrite);
# Helper function for cfgCommandWrite() to correctly format options for command-line usage
sub cfgCommandWriteOptionFormat
{
my $strOption = shift;
my $bMulti = shift;
my $bSecure = shift;
my $oValue = shift;
# Loops though all keys in the hash
my $strOptionFormat = '';
my $strParam;
foreach my $strKey (sort(keys(%$oValue)))
{
# Get the value - if the original value was a hash then the key must be prefixed
my $strValue = $bSecure ? '<redacted>' : ($bMulti ? "${strKey}=" : '') . $$oValue{$strKey};
# Handle the no- prefix for boolean values
if (cfgDefOptionType(cfgOptionId($strOption)) eq CFGDEF_TYPE_BOOLEAN)
{
$strParam = '--' . ($strValue ? '' : 'no-') . $strOption;
}
else
{
$strParam = "--${strOption}=${strValue}";
}
# Add quotes if the value has spaces in it
$strOptionFormat .= ' ' . (index($strValue, " ") != -1 ? "\"${strParam}\"" : $strParam);
}
return $strOptionFormat;
}
1;

64
lib/pgBackRest/LibC.pm
View File

@ -1,64 +0,0 @@
####################################################################################################################################
# C to Perl Interface
####################################################################################################################################
package pgBackRest::LibC;
use base 'Exporter';
use 5.010001;
use strict;
use warnings;
use Carp;
use pgBackRest::LibCAuto;
# Dynamically create constants
my $rhConstant = pgBackRest::LibCAuto::libcAutoConstant();
foreach my $strConstant (keys(%{$rhConstant}))
{
eval "use constant ${strConstant} => '" . $rhConstant->{$strConstant} . "'";
}
# Export functions and constants
our %EXPORT_TAGS = %{pgBackRest::LibCAuto::libcAutoExportTag()};
our @EXPORT_OK;
foreach my $strSection (keys(%EXPORT_TAGS))
{
# Assign values to serial constants like CFGCMD_* and CFGOPT_*. New commands and options (especially options) renumber the list
# and cause a lot of churn in the commits. This takes care of the renumbering to cut down on that churn.
my $strPrefixLast = 'XXXXXXXX';
my $iConstantIdx = 0;
foreach my $strConstant (@{$EXPORT_TAGS{$strSection}})
{
my $strPrefix = ($strConstant =~ m/^[A-Z0-9]+/g)[0];
if (defined($strPrefix))
{
if ($strPrefix ne $strPrefixLast)
{
$iConstantIdx = 0;
}
else
{
$iConstantIdx++;
}
if ($strPrefix eq 'CFGCMD' || $strPrefix eq 'CFGOPT')
{
eval "use constant ${strConstant} => ${iConstantIdx}";
}
$strPrefixLast = $strPrefix;
}
}
# OK to export everything in the tag
push(@EXPORT_OK, @{$EXPORT_TAGS{$strSection}});
}
# Nothing is exported by default
our @EXPORT = ();
1;

356
lib/pgBackRest/LibCAuto.pm
View File

@ -1,356 +0,0 @@
####################################################################################################################################
# Automatically generated by Build.pm -- do not modify directly.
####################################################################################################################################
package pgBackRest::LibCAuto;
use strict;
use warnings;
# Configuration option value constants
sub libcAutoConstant
{
return
{
CFGOPTVAL_COMPRESS_TYPE_NONE => 'none',
CFGOPTVAL_COMPRESS_TYPE_GZ => 'gz',
CFGOPTVAL_INFO_OUTPUT_TEXT => 'text',
CFGOPTVAL_INFO_OUTPUT_JSON => 'json',
CFGOPTVAL_REPO_LS_OUTPUT_TEXT => 'text',
CFGOPTVAL_REPO_LS_OUTPUT_JSON => 'json',
CFGOPTVAL_REMOTE_TYPE_PG => 'pg',
CFGOPTVAL_REMOTE_TYPE_REPO => 'repo',
CFGOPTVAL_REPO_CIPHER_TYPE_NONE => 'none',
CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc',
CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_FULL => 'full',
CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_DIFF => 'diff',
CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_INCR => 'incr',
CFGOPTVAL_REPO_S3_URI_STYLE_HOST => 'host',
CFGOPTVAL_REPO_S3_URI_STYLE_PATH => 'path',
CFGOPTVAL_REPO_TYPE_CIFS => 'cifs',
CFGOPTVAL_REPO_TYPE_POSIX => 'posix',
CFGOPTVAL_REPO_TYPE_S3 => 's3',
CFGOPTVAL_SORT_NONE => 'none',
CFGOPTVAL_SORT_ASC => 'asc',
CFGOPTVAL_SORT_DESC => 'desc',
CFGOPTVAL_RESTORE_TARGET_ACTION_PAUSE => 'pause',
CFGOPTVAL_RESTORE_TARGET_ACTION_PROMOTE => 'promote',
CFGOPTVAL_RESTORE_TARGET_ACTION_SHUTDOWN => 'shutdown',
CFGOPTVAL_BACKUP_TYPE_FULL => 'full',
CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff',
CFGOPTVAL_BACKUP_TYPE_INCR => 'incr',
CFGOPTVAL_RESTORE_TYPE_NAME => 'name',
CFGOPTVAL_RESTORE_TYPE_TIME => 'time',
CFGOPTVAL_RESTORE_TYPE_XID => 'xid',
CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve',
CFGOPTVAL_RESTORE_TYPE_NONE => 'none',
CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate',
CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default',
CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby',
CFGDEF_TYPE_BOOLEAN => 0,
CFGDEF_TYPE_FLOAT => 1,
CFGDEF_TYPE_HASH => 2,
CFGDEF_TYPE_INTEGER => 3,
CFGDEF_TYPE_LIST => 4,
CFGDEF_TYPE_PATH => 5,
CFGDEF_TYPE_SIZE => 6,
CFGDEF_TYPE_STRING => 7,
ENCODE_TYPE_BASE64 => 0,
CIPHER_MODE_ENCRYPT => 0,
CIPHER_MODE_DECRYPT => 1,
}
}
# Export function and constants
sub libcAutoExportTag
{
return
{
config =>
[
'CFGOPTVAL_COMPRESS_TYPE_NONE',
'CFGOPTVAL_COMPRESS_TYPE_GZ',
'CFGOPTVAL_INFO_OUTPUT_TEXT',
'CFGOPTVAL_INFO_OUTPUT_JSON',
'CFGOPTVAL_REPO_LS_OUTPUT_TEXT',
'CFGOPTVAL_REPO_LS_OUTPUT_JSON',
'CFGOPTVAL_REMOTE_TYPE_PG',
'CFGOPTVAL_REMOTE_TYPE_REPO',
'CFGOPTVAL_REPO_CIPHER_TYPE_NONE',
'CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC',
'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_FULL',
'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_DIFF',
'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_INCR',
'CFGOPTVAL_REPO_S3_URI_STYLE_HOST',
'CFGOPTVAL_REPO_S3_URI_STYLE_PATH',
'CFGOPTVAL_REPO_TYPE_CIFS',
'CFGOPTVAL_REPO_TYPE_POSIX',
'CFGOPTVAL_REPO_TYPE_S3',
'CFGOPTVAL_SORT_NONE',
'CFGOPTVAL_SORT_ASC',
'CFGOPTVAL_SORT_DESC',
'CFGOPTVAL_RESTORE_TARGET_ACTION_PAUSE',
'CFGOPTVAL_RESTORE_TARGET_ACTION_PROMOTE',
'CFGOPTVAL_RESTORE_TARGET_ACTION_SHUTDOWN',
'CFGOPTVAL_BACKUP_TYPE_FULL',
'CFGOPTVAL_BACKUP_TYPE_DIFF',
'CFGOPTVAL_BACKUP_TYPE_INCR',
'CFGOPTVAL_RESTORE_TYPE_NAME',
'CFGOPTVAL_RESTORE_TYPE_TIME',
'CFGOPTVAL_RESTORE_TYPE_XID',
'CFGOPTVAL_RESTORE_TYPE_PRESERVE',
'CFGOPTVAL_RESTORE_TYPE_NONE',
'CFGOPTVAL_RESTORE_TYPE_IMMEDIATE',
'CFGOPTVAL_RESTORE_TYPE_DEFAULT',
'CFGOPTVAL_RESTORE_TYPE_STANDBY',
'CFGCMD_ARCHIVE_GET',
'CFGCMD_ARCHIVE_PUSH',
'CFGCMD_BACKUP',
'CFGCMD_CHECK',
'CFGCMD_EXPIRE',
'CFGCMD_HELP',
'CFGCMD_INFO',
'CFGCMD_REPO_CREATE',
'CFGCMD_REPO_GET',
'CFGCMD_REPO_LS',
'CFGCMD_REPO_PUT',
'CFGCMD_REPO_RM',
'CFGCMD_RESTORE',
'CFGCMD_STANZA_CREATE',
'CFGCMD_STANZA_DELETE',
'CFGCMD_STANZA_UPGRADE',
'CFGCMD_START',
'CFGCMD_STOP',
'CFGCMD_VERSION',
'CFGOPT_ARCHIVE_ASYNC',
'CFGOPT_ARCHIVE_CHECK',
'CFGOPT_ARCHIVE_COPY',
'CFGOPT_ARCHIVE_GET_QUEUE_MAX',
'CFGOPT_ARCHIVE_PUSH_QUEUE_MAX',
'CFGOPT_ARCHIVE_TIMEOUT',
'CFGOPT_BACKUP_STANDBY',
'CFGOPT_BUFFER_SIZE',
'CFGOPT_CHECKSUM_PAGE',
'CFGOPT_CIPHER_PASS',
'CFGOPT_CMD_SSH',
'CFGOPT_COMPRESS',
'CFGOPT_COMPRESS_LEVEL',
'CFGOPT_COMPRESS_LEVEL_NETWORK',
'CFGOPT_COMPRESS_TYPE',
'CFGOPT_CONFIG',
'CFGOPT_CONFIG_INCLUDE_PATH',
'CFGOPT_CONFIG_PATH',
'CFGOPT_DB_INCLUDE',
'CFGOPT_DB_TIMEOUT',
'CFGOPT_DELTA',
'CFGOPT_EXCLUDE',
'CFGOPT_FILTER',
'CFGOPT_FORCE',
'CFGOPT_HOST_ID',
'CFGOPT_IGNORE_MISSING',
'CFGOPT_LINK_ALL',
'CFGOPT_LINK_MAP',
'CFGOPT_LOCK_PATH',
'CFGOPT_LOG_LEVEL_CONSOLE',
'CFGOPT_LOG_LEVEL_FILE',
'CFGOPT_LOG_LEVEL_STDERR',
'CFGOPT_LOG_PATH',
'CFGOPT_LOG_SUBPROCESS',
'CFGOPT_LOG_TIMESTAMP',
'CFGOPT_MANIFEST_SAVE_THRESHOLD',
'CFGOPT_NEUTRAL_UMASK',
'CFGOPT_ONLINE',
'CFGOPT_OUTPUT',
'CFGOPT_PG_HOST',
'CFGOPT_PG_HOST2',
'CFGOPT_PG_HOST3',
'CFGOPT_PG_HOST4',
'CFGOPT_PG_HOST5',
'CFGOPT_PG_HOST6',
'CFGOPT_PG_HOST7',
'CFGOPT_PG_HOST8',
'CFGOPT_PG_HOST_CMD',
'CFGOPT_PG_HOST_CMD2',
'CFGOPT_PG_HOST_CMD3',
'CFGOPT_PG_HOST_CMD4',
'CFGOPT_PG_HOST_CMD5',
'CFGOPT_PG_HOST_CMD6',
'CFGOPT_PG_HOST_CMD7',
'CFGOPT_PG_HOST_CMD8',
'CFGOPT_PG_HOST_CONFIG',
'CFGOPT_PG_HOST_CONFIG2',
'CFGOPT_PG_HOST_CONFIG3',
'CFGOPT_PG_HOST_CONFIG4',
'CFGOPT_PG_HOST_CONFIG5',
'CFGOPT_PG_HOST_CONFIG6',
'CFGOPT_PG_HOST_CONFIG7',
'CFGOPT_PG_HOST_CONFIG8',
'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH',
'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH2',
'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH3',
'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH4',
'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH5',
'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH6',
'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH7',
'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH8',
'CFGOPT_PG_HOST_CONFIG_PATH',
'CFGOPT_PG_HOST_CONFIG_PATH2',
'CFGOPT_PG_HOST_CONFIG_PATH3',
'CFGOPT_PG_HOST_CONFIG_PATH4',
'CFGOPT_PG_HOST_CONFIG_PATH5',
'CFGOPT_PG_HOST_CONFIG_PATH6',
'CFGOPT_PG_HOST_CONFIG_PATH7',
'CFGOPT_PG_HOST_CONFIG_PATH8',
'CFGOPT_PG_HOST_PORT',
'CFGOPT_PG_HOST_PORT2',
'CFGOPT_PG_HOST_PORT3',
'CFGOPT_PG_HOST_PORT4',
'CFGOPT_PG_HOST_PORT5',
'CFGOPT_PG_HOST_PORT6',
'CFGOPT_PG_HOST_PORT7',
'CFGOPT_PG_HOST_PORT8',
'CFGOPT_PG_HOST_USER',
'CFGOPT_PG_HOST_USER2',
'CFGOPT_PG_HOST_USER3',
'CFGOPT_PG_HOST_USER4',
'CFGOPT_PG_HOST_USER5',
'CFGOPT_PG_HOST_USER6',
'CFGOPT_PG_HOST_USER7',
'CFGOPT_PG_HOST_USER8',
'CFGOPT_PG_PATH',
'CFGOPT_PG_PATH2',
'CFGOPT_PG_PATH3',
'CFGOPT_PG_PATH4',
'CFGOPT_PG_PATH5',
'CFGOPT_PG_PATH6',
'CFGOPT_PG_PATH7',
'CFGOPT_PG_PATH8',
'CFGOPT_PG_PORT',
'CFGOPT_PG_PORT2',
'CFGOPT_PG_PORT3',
'CFGOPT_PG_PORT4',
'CFGOPT_PG_PORT5',
'CFGOPT_PG_PORT6',
'CFGOPT_PG_PORT7',
'CFGOPT_PG_PORT8',
'CFGOPT_PG_SOCKET_PATH',
'CFGOPT_PG_SOCKET_PATH2',
'CFGOPT_PG_SOCKET_PATH3',
'CFGOPT_PG_SOCKET_PATH4',
'CFGOPT_PG_SOCKET_PATH5',
'CFGOPT_PG_SOCKET_PATH6',
'CFGOPT_PG_SOCKET_PATH7',
'CFGOPT_PG_SOCKET_PATH8',
'CFGOPT_PG_USER',
'CFGOPT_PG_USER2',
'CFGOPT_PG_USER3',
'CFGOPT_PG_USER4',
'CFGOPT_PG_USER5',
'CFGOPT_PG_USER6',
'CFGOPT_PG_USER7',
'CFGOPT_PG_USER8',
'CFGOPT_PROCESS',
'CFGOPT_PROCESS_MAX',
'CFGOPT_PROTOCOL_TIMEOUT',
'CFGOPT_RAW',
'CFGOPT_RECOVERY_OPTION',
'CFGOPT_RECURSE',
'CFGOPT_REMOTE_TYPE',
'CFGOPT_REPO_CIPHER_PASS',
'CFGOPT_REPO_CIPHER_TYPE',
'CFGOPT_REPO_HARDLINK',
'CFGOPT_REPO_HOST',
'CFGOPT_REPO_HOST_CMD',
'CFGOPT_REPO_HOST_CONFIG',
'CFGOPT_REPO_HOST_CONFIG_INCLUDE_PATH',
'CFGOPT_REPO_HOST_CONFIG_PATH',
'CFGOPT_REPO_HOST_PORT',
'CFGOPT_REPO_HOST_USER',
'CFGOPT_REPO_PATH',
'CFGOPT_REPO_RETENTION_ARCHIVE',
'CFGOPT_REPO_RETENTION_ARCHIVE_TYPE',
'CFGOPT_REPO_RETENTION_DIFF',
'CFGOPT_REPO_RETENTION_FULL',
'CFGOPT_REPO_S3_BUCKET',
'CFGOPT_REPO_S3_CA_FILE',
'CFGOPT_REPO_S3_CA_PATH',
'CFGOPT_REPO_S3_ENDPOINT',
'CFGOPT_REPO_S3_HOST',
'CFGOPT_REPO_S3_KEY',
'CFGOPT_REPO_S3_KEY_SECRET',
'CFGOPT_REPO_S3_PORT',
'CFGOPT_REPO_S3_REGION',
'CFGOPT_REPO_S3_TOKEN',
'CFGOPT_REPO_S3_URI_STYLE',
'CFGOPT_REPO_S3_VERIFY_TLS',
'CFGOPT_REPO_TYPE',
'CFGOPT_RESUME',
'CFGOPT_SET',
'CFGOPT_SORT',
'CFGOPT_SPOOL_PATH',
'CFGOPT_STANZA',
'CFGOPT_START_FAST',
'CFGOPT_STOP_AUTO',
'CFGOPT_TABLESPACE_MAP',
'CFGOPT_TABLESPACE_MAP_ALL',
'CFGOPT_TARGET',
'CFGOPT_TARGET_ACTION',
'CFGOPT_TARGET_EXCLUSIVE',
'CFGOPT_TARGET_TIMELINE',
'CFGOPT_TYPE',
'cfgCommandName',
'cfgOptionIndexTotal',
'cfgOptionName',
],
configDefine =>
[
'CFGDEF_TYPE_BOOLEAN',
'CFGDEF_TYPE_FLOAT',
'CFGDEF_TYPE_HASH',
'CFGDEF_TYPE_INTEGER',
'CFGDEF_TYPE_LIST',
'CFGDEF_TYPE_PATH',
'CFGDEF_TYPE_SIZE',
'CFGDEF_TYPE_STRING',
'cfgCommandId',
'cfgDefOptionDefault',
'cfgDefOptionPrefix',
'cfgDefOptionSecure',
'cfgDefOptionType',
'cfgDefOptionValid',
'cfgOptionId',
'cfgOptionTotal',
],
debug =>
[
'libcUvSize',
],
storage =>
[
'storageRepoFree',
],
test =>
[
'cfgParseTest',
],
}
}
1;

79
lib/pgBackRest/Manifest.pm
View File

@ -18,8 +18,6 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
####################################################################################################################################
@ -94,31 +92,31 @@ use constant MANIFEST_KEY_TYPE => 'backup-t
push @EXPORT, qw(MANIFEST_KEY_TYPE);
# Options that were set when the backup was made
use constant MANIFEST_KEY_BACKUP_STANDBY => 'option-' . cfgOptionName(CFGOPT_BACKUP_STANDBY);
use constant MANIFEST_KEY_BACKUP_STANDBY => 'option-backup-standby';
push @EXPORT, qw(MANIFEST_KEY_BACKUP_STANDBY);
use constant MANIFEST_KEY_HARDLINK => 'option-hardlink';
push @EXPORT, qw(MANIFEST_KEY_HARDLINK);
use constant MANIFEST_KEY_ARCHIVE_CHECK => 'option-' . cfgOptionName(CFGOPT_ARCHIVE_CHECK);
use constant MANIFEST_KEY_ARCHIVE_CHECK => 'option-archive-check';
push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_CHECK);
use constant MANIFEST_KEY_ARCHIVE_COPY => 'option-' .cfgOptionName(CFGOPT_ARCHIVE_COPY);
use constant MANIFEST_KEY_ARCHIVE_COPY => 'option-archive-copy';
push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_COPY);
use constant MANIFEST_KEY_BUFFER_SIZE => 'option-' . cfgOptionName(CFGOPT_BUFFER_SIZE);
use constant MANIFEST_KEY_BUFFER_SIZE => 'option-buffer-size';
push @EXPORT, qw(MANIFEST_KEY_BUFFER_SIZE);
use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-' . cfgOptionName(CFGOPT_CHECKSUM_PAGE);
use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-checksum-page';
push @EXPORT, qw(MANIFEST_KEY_CHECKSUM_PAGE);
use constant MANIFEST_KEY_COMPRESS => 'option-' . cfgOptionName(CFGOPT_COMPRESS);
use constant MANIFEST_KEY_COMPRESS => 'option-compress';
push @EXPORT, qw(MANIFEST_KEY_COMPRESS);
use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-' . cfgOptionName(CFGOPT_COMPRESS_TYPE);
use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-compress-type';
push @EXPORT, qw(MANIFEST_KEY_COMPRESS_TYPE);
use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-' . cfgOptionName(CFGOPT_COMPRESS_LEVEL);
use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-compress-level';
push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL);
use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-' . cfgOptionName(CFGOPT_COMPRESS_LEVEL_NETWORK);
use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-compress-level-network';
push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL_NETWORK);
use constant MANIFEST_KEY_ONLINE => 'option-' . cfgOptionName(CFGOPT_ONLINE);
use constant MANIFEST_KEY_ONLINE => 'option-online';
push @EXPORT, qw(MANIFEST_KEY_ONLINE);
use constant MANIFEST_KEY_DELTA => 'option-' . cfgOptionName(CFGOPT_DELTA);
use constant MANIFEST_KEY_DELTA => 'option-delta';
push @EXPORT, qw(MANIFEST_KEY_DELTA);
use constant MANIFEST_KEY_PROCESS_MAX => 'option-' . cfgOptionName(CFGOPT_PROCESS_MAX);
use constant MANIFEST_KEY_PROCESS_MAX => 'option-process-max';
push @EXPORT, qw(MANIFEST_KEY_PROCESS_MAX);
# Information about the database that was backed up
@ -320,8 +318,8 @@ sub new
);
# Init object and store variables
my $self = $class->SUPER::new($strFileName, {bLoad => $bLoad, oStorage => $oStorage, strCipherPass => $strCipherPass,
strCipherPassSub => $strCipherPassSub});
my $self = $class->SUPER::new(
$oStorage, $strFileName, {bLoad => $bLoad, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub});
# If manifest not loaded from a file then the db version and catalog version must be set
if (!$bLoad)
@ -1111,9 +1109,6 @@ sub build
# consistent anyway and the one-second resolution problem is the least of our worries).
my $lTimeBegin = waitRemainder($bOnline);
# Check that links are valid
$self->linkCheck();
if (defined($oLastManifest))
{
$self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef,
@ -1244,52 +1239,6 @@ sub build
);
}
####################################################################################################################################
# linkCheck
#
# Check all link targets and make sure none of them are a subset of another link. In theory it would be possible to resolve the
# dependencies and generate a valid backup/restore but it's really complicated and there don't seem to be any compelling use cases.
####################################################################################################################################
sub linkCheck
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->linkCheck');
# Working variable
my $strBasePath = $self->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH);
foreach my $strTargetParent ($self->keys(MANIFEST_SECTION_BACKUP_TARGET))
{
if ($self->isTargetLink($strTargetParent))
{
my $strParentPath = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetParent, MANIFEST_SUBKEY_PATH);
my $strParentFile = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetParent, MANIFEST_SUBKEY_FILE, false);
foreach my $strTargetChild ($self->keys(MANIFEST_SECTION_BACKUP_TARGET))
{
if ($self->isTargetLink($strTargetChild) && $strTargetParent ne $strTargetChild)
{
my $strChildPath = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetChild, MANIFEST_SUBKEY_PATH);
my $strChildFile = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetParent, MANIFEST_SUBKEY_FILE, false);
if (!(defined($strParentFile) && defined($strChildFile)) &&
index(
storageLocal()->pathAbsolute($strBasePath, $strChildPath) . '/',
storageLocal()->pathAbsolute($strBasePath, $strParentPath) . '/') == 0)
{
confess &log(ERROR, 'link ' . $self->dbPathGet($strBasePath, $strTargetChild) .
" (${strChildPath}) references a subdirectory of or" .
" the same directory as link " . $self->dbPathGet($strBasePath, $strTargetParent) .
" (${strParentPath})", ERROR_LINK_DESTINATION);
}
}
}
}
}
}
####################################################################################################################################
# fileAdd
#

89
lib/pgBackRest/Protocol/Storage/Helper.pm
View File

@ -1,89 +0,0 @@
####################################################################################################################################
# Db & Repository Storage Helper
####################################################################################################################################
package pgBackRest::Protocol::Storage::Helper;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(basename);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::LibC qw(:storage);
use pgBackRest::Storage::Helper;
####################################################################################################################################
# Storage constants
####################################################################################################################################
use constant STORAGE_DB => '<DB>';
push @EXPORT, qw(STORAGE_DB);
use constant STORAGE_REPO => '<REPO>';
push @EXPORT, qw(STORAGE_REPO);
use constant STORAGE_REPO_ARCHIVE => '<REPO:ARCHIVE>';
push @EXPORT, qw(STORAGE_REPO_ARCHIVE);
use constant STORAGE_REPO_BACKUP => '<REPO:BACKUP>';
push @EXPORT, qw(STORAGE_REPO_BACKUP);
####################################################################################################################################
# Cache storage so it can be retrieved quickly
####################################################################################################################################
my $hStorage;
####################################################################################################################################
# storageRepo - get repository storage
####################################################################################################################################
sub storageRepo
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
) =
logDebugParam
(
__PACKAGE__ . '::storageRepo', \@_,
{name => 'strStanza', optional => true, trace => true},
);
# Create storage if not defined
if (!defined($hStorage->{&STORAGE_REPO}))
{
$hStorage->{&STORAGE_REPO} = new pgBackRest::Storage::Storage(
STORAGE_REPO, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)});
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}, trace => true},
);
}
push @EXPORT, qw(storageRepo);
####################################################################################################################################
# Clear the repo storage cache - FOR TESTING ONLY!
####################################################################################################################################
sub storageRepoCacheClear
{
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageRepoCacheClear');
delete($hStorage->{&STORAGE_REPO});
storageRepoFree();
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push @EXPORT, qw(storageRepoCacheClear);
1;

196
lib/pgBackRest/Storage/Base.pm
View File

@ -23,8 +23,8 @@ use pgBackRest::Common::Log;
use constant STORAGE_LOCAL => '<LOCAL>';
push @EXPORT, qw(STORAGE_LOCAL);
use constant STORAGE_S3 => 's3';
push @EXPORT, qw(STORAGE_S3);
use constant STORAGE_OBJECT => 'object';
push @EXPORT, qw(STORAGE_OBJECT);
use constant STORAGE_POSIX => 'posix';
push @EXPORT, qw(STORAGE_POSIX);
@ -71,145 +71,6 @@ sub new
);
}
####################################################################################################################################
# Copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed instead of
# file names.
####################################################################################################################################
sub copy
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xSourceFile,
$xDestinationFile,
$bSourceOpen,
) =
logDebugParam
(
__PACKAGE__ . '->copy', \@_,
{name => 'xSourceFile', required => false},
{name => 'xDestinationFile'},
{name => 'bSourceOpen', optional => true, default => false},
);
# Is source/destination an IO object or a file expression?
my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef;
# Does the source file exist?
my $bResult = false;
# Copy if the source file exists
if (defined($oSourceFileIo))
{
my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile);
# Use C copy if source and destination are C objects
if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite}))
{
$bResult = $self->{oStorageC}->copy(
$oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false;
}
else
{
# Open the source file if it is a C object
$bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true;
if ($bResult)
{
# Open the destination file if it is a C object
if (defined($oDestinationFileIo->{oStorageCWrite}))
{
$oDestinationFileIo->open();
}
# Copy the data
do
{
# Read data
my $tBuffer = '';
$oSourceFileIo->read(\$tBuffer, $self->{lBufferMax});
$oDestinationFileIo->write(\$tBuffer);
}
while (!$oSourceFileIo->eof());
# Close files
$oSourceFileIo->close();
$oDestinationFileIo->close();
}
}
}
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => $bResult, trace => true},
);
}
####################################################################################################################################
# get - reads a buffer from storage all at once
####################################################################################################################################
sub get
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFile,
$strCipherPass,
) =
logDebugParam
(
__PACKAGE__ . '->get', \@_,
{name => 'xFile', required => false, trace => true},
{name => 'strCipherPass', optional => true, redact => true},
);
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or
# if the repo has a user passphrase defined - else pass undef
my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead(
$xFile, {strCipherPass => defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()})) : undef;
# Read only if there is something to read from
my $tContent;
my $lSize = 0;
if (defined($oFileIo))
{
my $lSizeRead;
do
{
$lSizeRead = $oFileIo->read(\$tContent, $self->{lBufferMax});
$lSize += $lSizeRead;
}
while ($lSizeRead != 0);
# Close the file
$oFileIo->close();
# If nothing was read then set to undef
if ($lSize == 0)
{
$tContent = undef;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rtContent', value => defined($oFileIo) ? \$tContent : undef, trace => true},
);
}
####################################################################################################################################
# Calculate sha1 hash and size of file. If special encryption settings are required, then the file objects from openRead/openWrite
# must be passed instead of file names.
@ -328,57 +189,4 @@ sub pathAbsolute
);
}
####################################################################################################################################
# put - writes a buffer out to storage all at once
####################################################################################################################################
sub put
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFile,
$xContent,
$strCipherPass,
) =
logDebugParam
(
__PACKAGE__ . '->put', \@_,
{name => 'xFile', trace => true},
{name => 'xContent', required => false, trace => true},
{name => 'strCipherPass', optional => true, trace => true, redact => true},
);
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if
# the repo has a user passphrase defined - else pass undef
my $oFileIo = ref($xFile) ? $xFile : $self->openWrite(
$xFile, {strCipherPass => defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()});
# Determine size of content
my $lSize = defined($xContent) ? length(ref($xContent) ? $$xContent : $xContent) : 0;
# Write only if there is something to write
if ($lSize > 0)
{
$oFileIo->write(ref($xContent) ? $xContent : \$xContent);
}
# Else open the file so a zero length file is created (since file is not opened until first write)
else
{
$oFileIo->open();
}
# Close the file
$oFileIo->close();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'lSize', value => $lSize, trace => true},
);
}
1;

90
lib/pgBackRest/Storage/Helper.pm
View File

@ -1,5 +1,5 @@
####################################################################################################################################
# Local Storage Helper
# Repository Storage Helper
####################################################################################################################################
package pgBackRest::Storage::Helper;
@ -12,17 +12,9 @@ use Exporter qw(import);
use File::Basename qw(basename);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Storage;
use pgBackRest::Version;
####################################################################################################################################
# Compression extension
####################################################################################################################################
use constant COMPRESS_EXT => 'gz';
push @EXPORT, qw(COMPRESS_EXT);
####################################################################################################################################
# Temp file extension
####################################################################################################################################
@ -30,24 +22,88 @@ use constant STORAGE_TEMP_EXT => PROJECT_E
push @EXPORT, qw(STORAGE_TEMP_EXT);
####################################################################################################################################
# storageLocal - get local storage
#
# Local storage is generally read-only (except for locking) and can never reference a remote path. Used for adhoc activities like
# reading pgbackrest.conf.
# Cache storage so it can be retrieved quickly
####################################################################################################################################
sub storageLocal
my $oRepoStorage;
####################################################################################################################################
# storageRepoCommandSet
####################################################################################################################################
my $strStorageRepoCommand;
my $strStorageRepoType;
sub storageRepoCommandSet
{
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageLocal');
my
(
$strOperation,
$strCommand,
$strStorageType,
) =
logDebugParam
(
__PACKAGE__ . '::storageRepoCommandSet', \@_,
{name => 'strCommand'},
{name => 'strStorageType'},
);
$strStorageRepoCommand = $strCommand;
$strStorageRepoType = $strStorageType;
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push @EXPORT, qw(storageRepoCommandSet);
####################################################################################################################################
# storageRepo - get repository storage
####################################################################################################################################
sub storageRepo
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strStanza,
) =
logDebugParam
(
__PACKAGE__ . '::storageRepo', \@_,
{name => 'strStanza', optional => true, trace => true},
);
# Create storage if not defined
if (!defined($oRepoStorage))
{
$oRepoStorage = new pgBackRest::Storage::Storage($strStorageRepoCommand, $strStorageRepoType, 64 * 1024, 60);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oStorageLocal', value => new pgBackRest::Storage::Storage(STORAGE_LOCAL), trace => true},
{name => 'oStorageRepo', value => $oRepoStorage, trace => true},
);
}
push @EXPORT, qw(storageLocal);
push @EXPORT, qw(storageRepo);
####################################################################################################################################
# Clear the repo storage cache - FOR TESTING ONLY!
####################################################################################################################################
sub storageRepoCacheClear
{
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageRepoCacheClear');
undef($oRepoStorage);
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push @EXPORT, qw(storageRepoCacheClear);
1;

446
lib/pgBackRest/Storage/Storage.pm
View File

@ -17,10 +17,9 @@ use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Handle;
use pgBackRest::Common::Io::Process;
use pgBackRest::Common::Log;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::StorageRead;
use pgBackRest::Storage::StorageWrite;
####################################################################################################################################
# new
@ -36,32 +35,24 @@ sub new
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strCommand},
$self->{strType},
$self->{strPath},
$self->{lBufferMax},
$self->{iTimeoutIo},
$self->{strDefaultPathMode},
$self->{strDefaultFileMode},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strCommand'},
{name => 'strType'},
{name => 'strPath', optional => true},
{name => 'lBufferMax', optional => true, default => 65536},
{name => 'lBufferMax'},
{name => 'iTimeoutIo'},
{name => 'strDefaultPathMode', optional => true, default => '0750'},
{name => 'strDefaultFileMode', optional => true, default => '0640'},
);
# Create C storage object
$self->{oStorageC} = pgBackRest::LibC::Storage->new($self->{strType}, $self->{strPath});
# Get encryption settings
if ($self->{strType} eq '<REPO>')
{
$self->{strCipherType} = $self->{oStorageC}->cipherType();
$self->{strCipherPass} = $self->{oStorageC}->cipherPass();
}
# Create JSON object
$self->{oJSON} = JSON::PP->new()->allow_nonref();
@ -73,6 +64,103 @@ sub new
);
}
####################################################################################################################################
# Escape characteres that have special meaning on the command line
####################################################################################################################################
sub escape
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strValue,
) =
logDebugParam
(
__PACKAGE__ . '->escape', \@_,
{name => 'strValue', trace => true},
);
$strValue =~ s/\\/\\\\/g;
$strValue =~ s/\</\\\</g;
$strValue =~ s/\>/\\\>/g;
$strValue =~ s/\!/\\\!/g;
$strValue =~ s/\*/\\\*/g;
$strValue =~ s/\(/\\\(/g;
$strValue =~ s/\)/\\\)/g;
$strValue =~ s/\&/\\\&/g;
$strValue =~ s/\'/\\\'/g;
$strValue =~ s/\;/\\\;/g;
$strValue =~ s/\?/\\\?/g;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strValue', value => $strValue},
);
}
####################################################################################################################################
# Execute command and return the output
####################################################################################################################################
sub exec
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCommand,
) =
logDebugParam
(
__PACKAGE__ . '->exec', \@_,
{name => 'strCommand'},
);
$strCommand = "$self->{strCommand} ${strCommand}";
my $oBuffer = new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax});
my $oProcess = new pgBackRest::Common::Io::Process($oBuffer, $strCommand);
my $tResult;
while (!$oBuffer->eof())
{
$oBuffer->read(\$tResult, $self->{lBufferMax}, false);
}
$oProcess->close();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'tResult', value => $tResult},
{name => 'iExitStatus', value => $oProcess->exitStatus()},
);
}
####################################################################################################################################
# Create storage
####################################################################################################################################
sub create
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->create');
$self->exec("repo-create");
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# Check if file exists (not a path)
####################################################################################################################################
@ -96,7 +184,7 @@ sub exists
return logDebugReturn
(
$strOperation,
{name => 'bExists', value => defined($self->info($strFileExp, {bIgnoreMissing => true}))}
{name => 'bExists', value => $self->info($strFileExp, {bIgnoreMissing => true})->{type} eq 'f'}
);
}
@ -113,33 +201,49 @@ sub get
$strOperation,
$xFile,
$strCipherPass,
$bRaw,
) =
logDebugParam
(
__PACKAGE__ . '->get', \@_,
{name => 'xFile', required => false, trace => true},
{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true},
{name => 'xFile', required => false},
{name => 'strCipherPass', optional => true, redact => true},
{name => 'bRaw', optional => true, default => false},
);
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or
# if the repo has a user passphrase defined - else pass undef
my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead($xFile, {strCipherPass => $strCipherPass})) : undef;
# If openRead() was called first set values from that call
my $strFile = $xFile;
my $bIgnoreMissing = false;
# Get the file contents
my $bEmpty = false;
my $tContent = $self->{oStorageC}->get($oFileIo->{oStorageCRead});
if (defined($tContent) && length($tContent) == 0)
if (ref($xFile))
{
$tContent = undef;
$bEmpty = true;
$strFile = $xFile->{strFile};
$bIgnoreMissing = $xFile->{bIgnoreMissing};
$strCipherPass = $xFile->{strCipherPass};
}
# Check invalid params
if ($bRaw && defined($strCipherPass))
{
confess &log(ERROR, 'bRaw and strCipherPass cannot both be set');
}
# Get file
my ($tResult, $iExitStatus) = $self->exec(
(defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . ($bRaw ? ' --raw' : '') .
($bIgnoreMissing ? ' --ignore-missing' : '') . ' repo-get ' . $self->escape($strFile));
# Error if missing an not ignored
if ($iExitStatus == 1 && !$bIgnoreMissing)
{
confess &log(ERROR, "unable to open '${strFile}'", ERROR_FILE_OPEN);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rtContent', value => defined($tContent) || $bEmpty ? \$tContent : undef, trace => true},
{name => 'rtContent', value => $iExitStatus == 0 ? \$tResult : undef, trace => true},
);
}
@ -164,19 +268,11 @@ sub info
{name => 'bIgnoreMissing', optional => true, default => false},
);
my $rhInfo;
my $strJson = $self->{oStorageC}->info($strPathFileExp, $bIgnoreMissing);
if (defined($strJson))
{
$rhInfo = $self->{oJSON}->decode($strJson);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rhInfo', value => $rhInfo, trace => true}
{name => 'rhInfo', value => $self->manifest($strPathFileExp, {bRecurse => false})->{'.'}, trace => true}
);
}
@ -207,11 +303,14 @@ sub list
# Get file list
my $rstryFileList = [];
my $strFileList = $self->{oStorageC}->list($strPathExp, $bIgnoreMissing, $strSortOrder eq 'forward', $strExpression);
my $rhManifest = $self->manifest($strPathExp, {bRecurse => false});
if (defined($strFileList) && $strFileList ne '[]')
foreach my $strKey ($strSortOrder eq 'reverse' ? sort {$b cmp $a} keys(%{$rhManifest}) : sort keys(%{$rhManifest}))
{
$rstryFileList = $self->{oJSON}->decode($strFileList);
next if $strKey eq '.';
next if defined($strExpression) && $strKey !~ $strExpression;
push(@{$rstryFileList}, $strKey);
}
# Return from function and log return values if any
@ -234,50 +333,54 @@ sub manifest
(
$strOperation,
$strPathExp,
$strFilter,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->manifest', \@_,
{name => 'strPathExp'},
{name => 'strFilter', optional => true, trace => true},
{name => 'bRecurse', optional => true, default => true},
);
my $hManifest = $self->{oJSON}->decode($self->manifestJson($strPathExp, {strFilter => $strFilter}));
my $rhManifest = $self->{oJSON}->decode(
$self->exec("--output=json " . ($bRecurse ? ' --recurse' : '') . " repo-ls " . $self->escape($strPathExp)));
# Transform the manifest to the old format
foreach my $strKey (keys(%{$rhManifest}))
{
if ($rhManifest->{$strKey}{type} eq 'file')
{
$rhManifest->{$strKey}{type} = 'f';
if (defined($rhManifest->{$strKey}{time}))
{
$rhManifest->{$strKey}{modified_time} = $rhManifest->{$strKey}{time};
delete($rhManifest->{$strKey}{time});
}
}
elsif ($rhManifest->{$strKey}{type} eq 'path')
{
$rhManifest->{$strKey}{type} = 'd';
}
elsif ($rhManifest->{$strKey}{type} eq 'link')
{
$rhManifest->{$strKey}{type} = 'l';
}
elsif ($rhManifest->{$strKey}{type} eq 'special')
{
$rhManifest->{$strKey}{type} = 's';
}
else
{
confess "invalid file type '$rhManifest->{type}'";
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hManifest', value => $hManifest, trace => true}
);
}
sub manifestJson
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
$strFilter,
) =
logDebugParam
(
__PACKAGE__ . '->manifestJson', \@_,
{name => 'strPathExp'},
{name => 'strFilter', optional => true, trace => true},
);
my $strManifestJson = $self->{oStorageC}->manifest($strPathExp, $strFilter);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strManifestJson', value => $strManifestJson, trace => true}
{name => 'rhManifest', value => $rhManifest, trace => true}
);
}
@ -292,137 +395,24 @@ sub openRead
my
(
$strOperation,
$xFileExp,
$strFile,
$bIgnoreMissing,
$rhyFilter,
$strCipherPass,
) =
logDebugParam
(
__PACKAGE__ . '->openRead', \@_,
{name => 'xFileExp'},
{name => 'strFile'},
{name => 'bIgnoreMissing', optional => true, default => false},
{name => 'rhyFilter', optional => true},
{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true},
{name => 'strCipherPass', optional => true, redact => true},
);
# Open the file
my $oFileIo = pgBackRest::LibC::StorageRead->new($self->{oStorageC}, $xFileExp, $bIgnoreMissing);
# If cipher is set then decryption is the first filter applied to the read
if (defined($self->cipherType()))
{
$oFileIo->filterAdd(STORAGE_FILTER_CIPHER_BLOCK, $self->{oJSON}->encode([false, $self->cipherType(), $strCipherPass]));
}
# Apply any other filters
if (defined($rhyFilter))
{
foreach my $rhFilter (@{$rhyFilter})
{
$oFileIo->filterAdd(
$rhFilter->{strClass}, defined($rhFilter->{rxyParam}) ? $self->{oJSON}->encode($rhFilter->{rxyParam}) : undef);
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oFileIo', value => new pgBackRest::Storage::StorageRead($self, $oFileIo), trace => true},
);
}
####################################################################################################################################
# Open file for writing
####################################################################################################################################
sub openWrite
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFileExp,
$strMode,
$strUser,
$strGroup,
$lTimestamp,
$bAtomic,
$bPathCreate,
$rhyFilter,
$strCipherPass,
) =
logDebugParam
(
__PACKAGE__ . '->openWrite', \@_,
{name => 'xFileExp'},
{name => 'strMode', optional => true, default => $self->{strDefaultFileMode}},
{name => 'strUser', optional => true},
{name => 'strGroup', optional => true},
{name => 'lTimestamp', optional => true, default => '0'},
{name => 'bAtomic', optional => true, default => false},
{name => 'bPathCreate', optional => true, default => true},
{name => 'rhyFilter', optional => true},
{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true},
);
# Open the file
my $oFileIo = pgBackRest::LibC::StorageWrite->new(
$self->{oStorageC}, $xFileExp, oct($strMode), $strUser, $strGroup, $lTimestamp, $bAtomic, $bPathCreate);
# Apply any other filters
if (defined($rhyFilter))
{
foreach my $rhFilter (@{$rhyFilter})
{
$oFileIo->filterAdd(
$rhFilter->{strClass}, defined($rhFilter->{rxyParam}) ? $self->{oJSON}->encode($rhFilter->{rxyParam}) : undef);
}
}
# If cipher is set then encryption is the last filter applied to the write
if (defined($self->cipherType()))
{
$oFileIo->filterAdd(STORAGE_FILTER_CIPHER_BLOCK, $self->{oJSON}->encode([true, $self->cipherType(), $strCipherPass]));
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'oFileIo', value => new pgBackRest::Storage::StorageWrite($self, $oFileIo), trace => true},
);
}
####################################################################################################################################
# Resolve a path expression into an absolute path
####################################################################################################################################
sub pathGet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPathExp,
) =
logDebugParam
(
__PACKAGE__ . '->pathGet', \@_,
{name => 'strPathExp'},
);
# Check exists
my $strPath = $self->{oStorageC}->pathGet($strPathExp);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strPath', value => $strPath, trace => true}
{name => 'rhFileIo', value => {strFile => $strFile, bIgnoreMissing => $bIgnoreMissing, strCipherPass => $strCipherPass},
trace => true},
);
}
@ -437,19 +427,17 @@ sub pathRemove
my
(
$strOperation,
$strPathExp,
$bIgnoreMissing,
$strPath,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->pathRemove', \@_,
{name => 'strPathExp'},
{name => 'bIgnoreMissing', optional => true, default => true},
{name => 'strPath'},
{name => 'bRecurse', optional => true, default => false},
);
$self->{oStorageC}->pathRemove($strPathExp, $bIgnoreMissing, $bRecurse);
$self->exec("repo-rm " . ($bRecurse ? '--recurse ' : '') . $self->escape($strPath));
# Return from function and log return values if any
return logDebugReturn($strOperation);
@ -466,31 +454,54 @@ sub put
my
(
$strOperation,
$xFile,
$xContent,
$strFile,
$tContent,
$strCipherPass,
$bRaw,
) =
logDebugParam
(
__PACKAGE__ . '->put', \@_,
{name => 'xFile', trace => true},
{name => 'xContent', required => false, trace => true},
{name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), trace => true, redact => true},
{name => 'strFile'},
{name => 'tContent', required => false},
{name => 'strCipherPass', optional => true, redact => true},
{name => 'bRaw', optional => true, default => false},
);
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if
# the repo has a user passphrase defined - else pass undef
my $oFileIo = ref($xFile) ? $xFile : $self->openWrite($xFile, {strCipherPass => $strCipherPass});
# Check invalid params
if ($bRaw && defined($strCipherPass))
{
confess &log(ERROR, 'bRaw and strCipherPass cannot both be set');
}
# Write the content
my $lSize = $self->{oStorageC}->put($oFileIo->{oStorageCWrite}, ref($xContent) ? $$xContent : $xContent);
# Put file
my $strCommand =
"$self->{strCommand}" . (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') .
($bRaw ? ' --raw' : '') . ' repo-put ' . $self->escape($strFile);
my $oBuffer = new pgBackRest::Common::Io::Buffered(
new pgBackRest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax});
my $oProcess = new pgBackRest::Common::Io::Process($oBuffer, $strCommand);
if (defined($tContent))
{
$oBuffer->write(\$tContent);
}
close($oBuffer->handleWrite());
my $tResult;
while (!$oBuffer->eof())
{
$oBuffer->read(\$tResult, $self->{lBufferMax}, false);
}
close($oBuffer->handleRead());
$oProcess->close();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'lSize', value => $lSize, trace => true},
);
return logDebugReturn($strOperation);
}
####################################################################################################################################
@ -504,20 +515,15 @@ sub remove
my
(
$strOperation,
$xFileExp,
$bIgnoreMissing,
$strFile,
) =
logDebugParam
(
__PACKAGE__ . '->remove', \@_,
{name => 'xFileExp'},
{name => 'bIgnoreMissing', optional => true, default => true},
);
foreach my $strFileExp (ref($xFileExp) ? @{$xFileExp} : ($xFileExp))
{
$self->{oStorageC}->remove($strFileExp, $bIgnoreMissing);
}
$self->exec("repo-rm " . $self->escape($strFile));
# Return from function and log return values if any
return logDebugReturn($strOperation);
@ -527,8 +533,6 @@ sub remove
# Getters
####################################################################################################################################
sub capability {shift->type() eq STORAGE_POSIX}
sub type {shift->{oStorageC}->type()}
sub cipherType {shift->{strCipherType}}
sub cipherPassUser {shift->{strCipherPass}}
sub type {shift->{strType}}
1;

143
lib/pgBackRest/Storage/StorageRead.pm
View File

@ -1,143 +0,0 @@
####################################################################################################################################
# C Storage Read Interface
####################################################################################################################################
package pgBackRest::Storage::StorageRead;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(dirname);
use Fcntl qw(:mode);
use File::stat qw{lstat};
use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Storage::Base;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift;
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{oStorage},
$self->{oStorageCRead},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oStorage'},
{name => 'oStorageCRead'},
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# Open the file
####################################################################################################################################
sub open
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->open');
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => $self->{oStorageCRead}->open() ? true : false, trace => true},
);
}
####################################################################################################################################
# Read data
####################################################################################################################################
sub read
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my (
$strOperation,
$rtBuffer,
$iSize,
) =
logDebugParam
(
__PACKAGE__ . '->read', \@_,
{name => 'rtBuffer'},
{name => 'iSize'},
);
# Read if not eof
my $iActualSize = 0;
if (!$self->eof())
{
my $tBuffer = $self->{oStorageCRead}->read($iSize);
$iActualSize = length($tBuffer);
$$rtBuffer .= $tBuffer;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iActualSize', value => $iActualSize}
);
}
####################################################################################################################################
# Is the file at eof?
####################################################################################################################################
sub eof
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->eof');
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => $self->{oStorageCRead}->eof() ? true : false, trace => true},
);
}
####################################################################################################################################
# Close the file
####################################################################################################################################
sub close
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->close');
$self->{oStorageCRead}->close();
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => true, trace => true},
);
}
1;

116
lib/pgBackRest/Storage/StorageWrite.pm
View File

@ -1,116 +0,0 @@
####################################################################################################################################
# C Storage Write Interface
####################################################################################################################################
package pgBackRest::Storage::StorageWrite;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(dirname);
use Fcntl qw(:mode);
use File::stat qw{lstat};
use JSON::PP;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Storage::Base;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift;
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{oStorage},
$self->{oStorageCWrite},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oStorage'},
{name => 'oStorageCWrite'},
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# Open the file
####################################################################################################################################
sub open
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->open');
$self->{oStorageCWrite}->open();
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => true, trace => true},
);
}
####################################################################################################################################
# Write data
####################################################################################################################################
sub write
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my (
$strOperation,
$rtBuffer,
) =
logDebugParam
(
__PACKAGE__ . '->write', \@_,
{name => 'rtBuffer'},
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iActualSize', value => $self->{oStorageCWrite}->write($$rtBuffer)}
);
}
####################################################################################################################################
# Close the file
####################################################################################################################################
sub close
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->close');
$self->{oStorageCWrite}->close();
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => true, trace => true},
);
}
1;

78
libc/LibC.h
View File

@ -1,78 +0,0 @@
/***********************************************************************************************************************************
Helper macros for LibC.xs
***********************************************************************************************************************************/
/***********************************************************************************************************************************
Package Names
***********************************************************************************************************************************/
#define PACKAGE_NAME "pgBackRest"
#define PACKAGE_NAME_LIBC PACKAGE_NAME "::LibC"
/***********************************************************************************************************************************
Error handling macros that throw a Perl error when a C error is caught
***********************************************************************************************************************************/
#define ERROR_XS_BEGIN() \
TRY_BEGIN()
#define ERROR_XS() \
croak("PGBRCLIB:%d:%s:%d:%s", errorCode(), errorFileName(), errorFileLine(), errorMessage());
#define ERROR_XS_END() \
CATCH_ANY() \
{ \
ERROR_XS(); \
} \
TRY_END();
/***********************************************************************************************************************************
Simplifies switching to a temp memory context in functions and includes error handling
***********************************************************************************************************************************/
#define MEM_CONTEXT_XS_TEMP() \
MEM_CONTEXT_XS_TEMP_memContext
#define MEM_CONTEXT_XS_TEMP_BEGIN() \
{ \
/* Create temp memory context */ \
MemContext *MEM_CONTEXT_XS_TEMP() = memContextNew("temporary"); \
\
/* Switch to temp memory context */ \
memContextSwitch(MEM_CONTEXT_XS_TEMP()); \
\
/* Store any errors to be croaked to Perl at the end */ \
bool MEM_CONTEXT_XS_croak = false; \
\
/* Try the statement block */ \
TRY_BEGIN()
#define MEM_CONTEXT_XS_TEMP_END() \
/* Set error to be croak to Perl later */ \
CATCH_ANY() \
{ \
MEM_CONTEXT_XS_croak = true; \
} \
/* Free the context on error */ \
FINALLY() \
{ \
memContextSwitchBack(); \
memContextDiscard(); \
} \
TRY_END(); \
\
/* Croak on error */ \
if (MEM_CONTEXT_XS_croak) \
{ \
ERROR_XS() \
} \
}
/***********************************************************************************************************************************
Create new string from an SV
***********************************************************************************************************************************/
#define STR_NEW_SV(param) \
(SvOK(param) ? strNewN(SvPV_nolen(param), SvCUR(param)) : NULL)
/***********************************************************************************************************************************
Create const buffer from an SV
***********************************************************************************************************************************/
#define BUF_CONST_SV(param) \
(SvOK(param) ? BUF(SvPV_nolen(param), SvCUR(param)) : NULL)

98
libc/LibC.xs
View File

@ -1,98 +0,0 @@
/***********************************************************************************************************************************
C to Perl Interface
The following C types are mapped by the current typemap:
'AV *', 'Boolean', 'CV *', 'FILE *', 'FileHandle', 'HV *', 'I16', 'I32', 'I8', 'IV', 'InOutStream', 'InputStream', 'NV',
'OutputStream', 'PerlIO *', 'Result', 'STRLEN', 'SV *', 'SVREF', 'SysRet', 'SysRetLong', 'Time_t *', 'U16', 'U32', 'U8', 'UV',
'bool', 'bool_t', 'caddr_t', 'char', 'char *', 'char **', 'const char *', 'double', 'float', 'int', 'long', 'short', 'size_t',
'ssize_t', 'time_t', 'unsigned', 'unsigned char', 'unsigned char *', 'unsigned int', 'unsigned long', 'unsigned long *',
'unsigned short', 'void *', 'wchar_t', 'wchar_t *'
***********************************************************************************************************************************/
#include "build.auto.h"
#define PERL_NO_GET_CONTEXT
/***********************************************************************************************************************************
Perl includes
Order is critical here so don't change it.
***********************************************************************************************************************************/
#if __GNUC__ > 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ > 8 || (__GNUC_MINOR__ == 8 && __GNUC_PATCHLEVEL__ >= 0)))
#define WARNING_MAYBE_INITIALIZED 1
#elif __GNUC__ > 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ > 6 || (__GNUC_MINOR__ == 6 && __GNUC_PATCHLEVEL__ >= 0)))
#define WARNING_INITIALIZED 1
#endif
#if WARNING_MAYBE_INITIALIZED
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#elif WARNING_INITIALIZED
#pragma GCC diagnostic ignored "-Wuninitialized"
#endif
#pragma GCC diagnostic ignored "-Wsign-conversion"
#pragma GCC diagnostic ignored "-Wconversion"
#include <XSUB.h>
#include <EXTERN.h>
#include <perl.h>
#if WARNING_MAYBE_INITIALIZED
#pragma GCC diagnostic warning "-Wmaybe-uninitialized"
#elif WARNING_INITIALIZED
#pragma GCC diagnostic warning "-Wuninitialized"
#endif
/***********************************************************************************************************************************
C includes
These includes are from the src directory. There is no Perl-specific code in them.
***********************************************************************************************************************************/
#include "common/error.h"
#include "common/io/io.h"
#include "config/config.h"
#include "config/define.h"
#include "config/load.h"
#include "config/parse.h"
#include "storage/posix/storage.h"
/***********************************************************************************************************************************
Helper macros
***********************************************************************************************************************************/
#include "LibC.h"
/***********************************************************************************************************************************
XSH includes
These includes define data structures that are required for the C to Perl interface but are not part of the regular C source.
***********************************************************************************************************************************/
#include "xs/config/configTest.xsh"
#include "xs/storage/storage.xsh"
#include "xs/storage/storageRead.xsh"
#include "xs/storage/storageWrite.xsh"
/***********************************************************************************************************************************
Module definition
***********************************************************************************************************************************/
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC
PROTOTYPES: DISABLE
# Return UVSIZE to ensure that this Perl supports 64-bit integers
# ----------------------------------------------------------------------------------------------------------------------------------
I32
libcUvSize()
CODE:
RETVAL = UVSIZE;
OUTPUT:
RETVAL
# Exported functions and modules
#
# These modules should map 1-1 with C modules in src directory.
# ----------------------------------------------------------------------------------------------------------------------------------
INCLUDE: xs/config/config.xs
INCLUDE: xs/config/configTest.xs
INCLUDE: xs/config/define.xs
INCLUDE: xs/storage/storage.xs
INCLUDE: xs/storage/storageRead.xs
INCLUDE: xs/storage/storageWrite.xs

147
libc/Makefile.PL
View File

@ -1,147 +0,0 @@
####################################################################################################################################
# Build Makefile and Auto-Generate Files Required for Build
#
# The C library is only used for Perl unit tests. For a production build all C library exports are built directly into the
# pgbackrest executable. See src/perl/libc.auto.c.
####################################################################################################################################
use 5.010001;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
# Convert die to confess to capture the stack trace
$SIG{__DIE__} = sub { Carp::confess @_ };
use Cwd qw(abs_path);
use ExtUtils::MakeMaker;
use File::Basename qw(dirname);
####################################################################################################################################
# Storage object to use for all file operations
####################################################################################################################################
use constant PROJECT_NAME => 'pgBackRest';
####################################################################################################################################
# Make sure the makefile is being created in an expected test directory. This should prevent users from building it in production.
####################################################################################################################################
if (dirname($0) !~ /\.vagrant\/bin\/[^\/]+\/libc$/)
{
confess
"LibC is not being built in a test directory (" . dirname($0) .
"). LibC should not be distributed for production builds. See build documentation for details.";
}
####################################################################################################################################
# Create C Makefile
####################################################################################################################################
my $strBuildPath = dirname(dirname(abs_path($0)));
# Create C files array
my @stryCFile =
(
'LibC.c',
'command/command.c',
'common/compress/gz/common.c',
'common/compress/gz/compress.c',
'common/compress/gz/decompress.c',
'common/compress/helper.c',
'common/crypto/cipherBlock.c',
'common/crypto/common.c',
'common/crypto/hash.c',
'common/debug.c',
'common/encode.c',
'common/encode/base64.c',
'common/error.c',
'common/ini.c',
'common/io/bufferRead.c',
'common/io/bufferWrite.c',
'common/io/filter/buffer.c',
'common/io/filter/filter.c',
'common/io/filter/group.c',
'common/io/filter/sink.c',
'common/io/filter/size.c',
'common/io/handleWrite.c',
'common/io/http/cache.c',
'common/io/http/client.c',
'common/io/http/common.c',
'common/io/http/header.c',
'common/io/http/query.c',
'common/io/io.c',
'common/io/read.c',
'common/io/tls/client.c',
'common/io/write.c',
'common/log.c',
'common/memContext.c',
'common/regExp.c',
'common/stackTrace.c',
'common/time.c',
'common/type/convert.c',
'common/type/buffer.c',
'common/type/json.c',
'common/type/keyValue.c',
'common/type/list.c',
'common/type/string.c',
'common/type/stringList.c',
'common/type/variant.c',
'common/type/variantList.c',
'common/type/xml.c',
'common/user.c',
'common/wait.c',
'config/config.c',
'config/define.c',
'config/load.c',
'config/parse.c',
'protocol/client.c',
'protocol/command.c',
'protocol/helper.c',
'protocol/parallel.c',
'protocol/parallelJob.c',
'protocol/server.c',
'storage/posix/read.c',
'storage/posix/storage.c',
'storage/posix/write.c',
'storage/s3/read.c',
'storage/s3/storage.c',
'storage/s3/write.c',
'storage/helper.c',
'storage/read.c',
'storage/storage.c',
'storage/write.c',
);
# Add ../src for files that are outside libc
for (my $iFileIdx = 1; $iFileIdx < @stryCFile; $iFileIdx++)
{
$stryCFile[$iFileIdx] = '../src/' . $stryCFile[$iFileIdx];
}
# Write the makefile
WriteMakefile
(
NAME => PROJECT_NAME . '::LibC',
VERSION => '999',
AUTHOR => 'David Steele <david@pgbackrest.org>',
CCFLAGS => join(' ', qw)
-Wfatal-errors -Wall -Wextra -Wwrite-strings -Wno-clobbered -Wno-missing-field-initializers
-o $@
-std=c99
-D_POSIX_C_SOURCE=200112L
-D_FILE_OFFSET_BITS=64
`pkg-config libxml-2.0 --cflags`
-I`pg_config --includedir`
)),
INC => join(' ', qw(
-I.
-I../src
)),
C => \@stryCFile,
LIBS => '-lcrypto -lpq -lssl -lxml2',
OBJECT => '$(O_FILES)',
);

372
libc/build/lib/pgBackRestLibC/Build.pm
View File

@ -1,372 +0,0 @@
####################################################################################################################################
# Auto-Generate XS and PM Files Required for Perl
####################################################################################################################################
package pgBackRestLibC::Build;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use Storable qw(dclone);
use lib dirname($0) . '/../build/lib';
use lib dirname($0) . '/../lib';
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Version;
use pgBackRestBuild::Build;
use pgBackRestBuild::Build::Common;
use pgBackRestBuild::Config::Data;
use pgBackRestBuild::Error::Data;
use pgBackRestTest::Common::Storage;
use pgBackRestTest::Common::StoragePosix;
####################################################################################################################################
# Perl function and constant exports
####################################################################################################################################
use constant BLD_EXPORTTYPE_SUB => 'sub';
use constant BLD_EXPORTTYPE_CONSTANT => 'constant';
use constant LIB_AUTO_NAME => 'LibCAuto';
####################################################################################################################################
# Save contents to a file if the file is missing or the contents are different. This saves write IO and prevents the timestamp from
# changing.
####################################################################################################################################
sub buildPutDiffers
{
my $oStorage = shift;
my $strFile = shift;
my $strContents = shift;
# Attempt to load the file
my $bSave = true;
my $oFile = $oStorage->openRead($strFile, {bIgnoreMissing => true});
# If file was found see if the content is the same
if (defined($oFile) && ${$oStorage->get($oFile)} eq $strContents)
{
$bSave = false;
}
# Save if the contents are different or missing
if ($bSave)
{
$oStorage->put($strFile, $strContents);
}
return $bSave;
}
####################################################################################################################################
# Static exports
####################################################################################################################################
my $rhExport =
{
'config' =>
{
&BLD_EXPORTTYPE_SUB => [qw(
cfgCommandName
cfgOptionIndexTotal
cfgOptionName
)],
},
'configDefine' =>
{
&BLD_EXPORTTYPE_SUB => [qw(
cfgCommandId
cfgDefOptionDefault
cfgDefOptionPrefix
cfgDefOptionSecure
cfgDefOptionType
cfgDefOptionValid
cfgOptionId
cfgOptionTotal
)],
},
'debug' =>
{
&BLD_EXPORTTYPE_SUB => [qw(
libcUvSize
)],
},
'storage' =>
{
&BLD_EXPORTTYPE_SUB => [qw(
storageRepoFree
)],
},
'test' =>
{
&BLD_EXPORTTYPE_SUB => [qw(
cfgParseTest
)],
},
};
####################################################################################################################################
# Execute all build functions
####################################################################################################################################
sub buildXsAll
{
my $strBuildPath = shift;
# List of files built
my @stryBuilt;
# Storage
my $oStorage = new pgBackRestTest::Common::Storage(
$strBuildPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false}));
# Build interface file
my $strContent =
('#' x 132) . "\n" .
'# ' . bldAutoWarning('Build.pm') . "\n" .
('#' x 132) . "\n" .
"package pgBackRest::LibCAuto;\n" .
"\n" .
"use strict;\n" .
"use warnings;\n";
# Generate constants for options that have a list of strings as allowed values
my $rhConfigDefine = cfgDefine();
my $strConstantBlock;
foreach my $strOption (sort(keys(%{$rhConfigDefine})))
{
my $rhOption = $rhConfigDefine->{$strOption};
next if $rhOption->{&CFGDEF_TYPE} ne CFGDEF_TYPE_STRING;
next if $strOption =~ /^log-level-/;
if (defined($rhOption->{&CFGDEF_ALLOW_LIST}))
{
$strConstantBlock .= defined($strConstantBlock) ? "\n" : '';
foreach my $strValue (@{$rhOption->{&CFGDEF_ALLOW_LIST}})
{
my $strConstant = 'CFGOPTVAL_' . uc("${strOption}_${strValue}");
$strConstant =~ s/\-/\_/g;
$strConstantBlock .= " ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> '${strValue}',\n";
push(@{$rhExport->{'config'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant);
}
}
foreach my $strCommand (sort(keys(%{$rhOption->{&CFGDEF_COMMAND}})))
{
my $rhCommand = $rhOption->{&CFGDEF_COMMAND}{$strCommand};
if (defined($rhCommand->{&CFGDEF_ALLOW_LIST}))
{
$strConstantBlock .= defined($strConstantBlock) ? "\n" : '';
foreach my $strValue (@{$rhCommand->{&CFGDEF_ALLOW_LIST}})
{
my $strConstant = 'CFGOPTVAL_' . uc("${strCommand}_${strOption}_${strValue}");
$strConstant =~ s/\-/\_/g;
$strConstantBlock .= " ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> '${strValue}',\n";
push(@{$rhExport->{'config'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant);
}
}
}
}
# Generate command constants
foreach my $strCommand (cfgDefineCommandList())
{
my $strConstant = "CFGCMD_" . uc($strCommand);
$strConstant =~ s/\-/\_/g;
push(@{$rhExport->{'config'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant);
}
# Generate option constants
foreach my $strOption (sort(keys(%{$rhConfigDefine})))
{
# Build Perl constant
my $strConstant = "CFGOPT_" . uc($strOption);
$strConstant =~ s/\-/\_/g;
# Builds option data
for (my $iOptionIndex = 1; $iOptionIndex <= $rhConfigDefine->{$strOption}{&CFGDEF_INDEX_TOTAL}; $iOptionIndex++)
{
push(@{$rhExport->{'config'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant . ($iOptionIndex == 1 ? '' : $iOptionIndex));
}
}
# Generate option type constants
$strConstantBlock .= defined($strConstantBlock) ? "\n" : '';
my $iIndex = 0;
foreach my $strOptionType (cfgDefineOptionTypeList())
{
# Build Perl constant
my $strConstant = "CFGDEF_TYPE_" . uc($strOptionType);
$strConstant =~ s/\-/\_/g;
$strConstantBlock .=
" ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> $iIndex,\n";
push(@{$rhExport->{'configDefine'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant);
$iIndex++;
};
# Generate encode type constants
$strConstantBlock .= defined($strConstantBlock) ? "\n" : '';
my $strConstant = "ENCODE_TYPE_BASE64";
$strConstantBlock .=
" ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> 0,\n";
# Generate cipher type constants
$strConstantBlock .= defined($strConstantBlock) ? "\n" : '';
$strConstant = "CIPHER_MODE_ENCRYPT";
$strConstantBlock .=
" ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> 0,\n";
$strConstant = "CIPHER_MODE_DECRYPT";
$strConstantBlock .=
" ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> 1,\n";
$strContent .=
"\n" .
"# Configuration option value constants\n" .
"sub libcAutoConstant\n" .
"{\n" .
" return\n" .
" {\n" .
" " . trim($strConstantBlock) . "\n" .
" }\n" .
"}\n";
# Generate export tags
my $strExportTags;
foreach my $strSection (sort(keys(%{$rhExport})))
{
my $rhExportSection = $rhExport->{$strSection};
$strExportTags .= (defined($strExportTags) ? "\n" : '') . " ${strSection} =>\n [\n";
if (defined($rhExportSection->{&BLD_EXPORTTYPE_CONSTANT}) && @{$rhExportSection->{&BLD_EXPORTTYPE_CONSTANT}} > 0)
{
foreach my $strConstant (@{$rhExportSection->{&BLD_EXPORTTYPE_CONSTANT}})
{
$strExportTags .= " '${strConstant}',\n";
}
}
if (defined($rhExportSection->{&BLD_EXPORTTYPE_SUB}) && @{$rhExportSection->{&BLD_EXPORTTYPE_SUB}} > 0)
{
foreach my $strSub (@{$rhExportSection->{&BLD_EXPORTTYPE_SUB}})
{
$strExportTags .= " '${strSub}',\n";
}
}
$strExportTags .= " ],\n";
}
$strContent .=
"\n" .
"# Export function and constants\n" .
"sub libcAutoExportTag\n" .
"{\n" .
" return\n" .
" {\n" .
$strExportTags .
" }\n" .
"}\n";
# Add module end marker
$strContent .=
"\n" .
"1;\n";
# Save the file
my $strFile = 'lib/' . PROJECT_NAME . '/' . LIB_AUTO_NAME . '.pm';
if (buildPutDiffers($oStorage, "../${strFile}", $strContent))
{
push(@stryBuilt, $strFile);
}
# Build error file
#-------------------------------------------------------------------------------------------------------------------------------
my $rhErrorDefine = errorDefine();
# Order by id for the list that is id ordered
my $rhErrorId = {};
foreach my $strType (sort(keys(%{$rhErrorDefine})))
{
my $iCode = $rhErrorDefine->{$strType};
$rhErrorId->{$iCode} = $strType;
}
# Output errors
$strContent =
('#' x 132) . "\n" .
"# COMMON EXCEPTION AUTO MODULE\n" .
"# \n" .
'# ' . bldAutoWarning('Build.pm') . "\n" .
('#' x 132) . "\n" .
"package pgBackRest::Common::ExceptionAuto;\n" .
"\n" .
"use strict;\n" .
"use warnings FATAL => qw(all);\n" .
"\n" .
"use Exporter qw(import);\n" .
" our \@EXPORT = qw();\n" .
"\n" .
('#' x 132) . "\n" .
"# Error Definitions\n" .
('#' x 132) . "\n" .
"use constant ERROR_MINIMUM => " . ERRDEF_MIN . ";\n" .
"push \@EXPORT, qw(ERROR_MINIMUM);\n" .
"use constant ERROR_MAXIMUM => " . ERRDEF_MAX . ";\n" .
"push \@EXPORT, qw(ERROR_MAXIMUM);\n" .
"\n";
foreach my $iCode (sort({sprintf("%03d", $a) cmp sprintf("%03d", $b)} keys(%{$rhErrorId})))
{
my $strType = "ERROR_" . uc($rhErrorId->{$iCode});
$strType =~ s/\-/\_/g;
$strContent .=
"use constant ${strType}" . (' ' x (54 - length($strType))) . " => $iCode;\n" .
"push \@EXPORT, qw(${strType});\n"
}
$strContent .=
"\n" .
"1;\n";
$strFile = 'lib/' . PROJECT_NAME . '/Common/ExceptionAuto.pm';
if (buildPutDiffers($oStorage, "../${strFile}", $strContent))
{
push(@stryBuilt, $strFile);
}
# Return list of files built
return @stryBuilt;
}
push @EXPORT, qw(buildXsAll);
1;

3
libc/typemap
View File

@ -1,3 +0,0 @@
pgBackRest::LibC::Storage T_PTROBJ
pgBackRest::LibC::StorageRead T_PTROBJ
pgBackRest::LibC::StorageWrite T_PTROBJ

17
libc/xs/config/config.xs
View File

@ -1,17 +0,0 @@
# ----------------------------------------------------------------------------------------------------------------------------------
# Config Perl Exports
# ----------------------------------------------------------------------------------------------------------------------------------
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC
const char *
cfgCommandName(commandId)
U32 commandId
I32
cfgOptionIndexTotal(optionId)
U32 optionId
const char *
cfgOptionName(optionId)
U32 optionId

38
libc/xs/config/configTest.xs
View File

@ -1,38 +0,0 @@
# ----------------------------------------------------------------------------------------------------------------------------------
# Config Test Perl Exports
# ----------------------------------------------------------------------------------------------------------------------------------
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC
####################################################################################################################################
# Parse command line and return a JSON object with results
####################################################################################################################################
SV *
cfgParseTest(backrestBin, parseParam)
const char *backrestBin
const char *parseParam
CODE:
RETVAL = NULL;
ERROR_XS_BEGIN()
{
// This should run in a temp context but for some reason getopt_long gets upset when if gets called again after the previous
// arg list being freed. So, this is a memory leak but it is only used for testing, not production.
StringList *paramList = strLstNewSplitZ(strCat(strNew("pgbackrest|"), parseParam), "|");
// Don't use cfgLoad() because it has a lot of side effects that we don't want
configParse(strLstSize(paramList), strLstPtr(paramList), false);
cfgExeSet(strNew(backrestBin));
cfgLoadUpdateOption();
String *result = perlOptionJson();
RETVAL = newSV(strSize(result));
SvPOK_only(RETVAL);
strcpy(SvPV_nolen(RETVAL), strPtr(result));
SvCUR_set(RETVAL, strSize(result));
}
ERROR_XS_END()
OUTPUT:
RETVAL

122
libc/xs/config/configTest.xsh
View File

@ -1,122 +0,0 @@
/***********************************************************************************************************************************
Config XS Header
***********************************************************************************************************************************/
#include <string.h>
#include "common/memContext.h"
#include "common/type/json.h"
#include "config/config.h"
/***********************************************************************************************************************************
Build JSON output from options
***********************************************************************************************************************************/
String *
perlOptionJson(void)
{
String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
KeyValue *configKv = kvNew();
for (ConfigOption optionId = 0; optionId < CFG_OPTION_TOTAL; optionId++)
{
// Skip if not valid
if (!cfgOptionValid(optionId))
continue;
Variant *optionVar = varNewKv(kvNew());
// Add valid
kvPut(varKv(optionVar), VARSTRDEF("valid"), BOOL_TRUE_VAR);
// Add source
const Variant *source = NULL;
switch (cfgOptionSource(optionId))
{
case cfgSourceParam:
{
source = varNewStrZ("param");
break;
}
case cfgSourceConfig:
{
source = varNewStrZ("config");
break;
}
case cfgSourceDefault:
{
source = varNewStrZ("default");
break;
}
}
kvPut(varKv(optionVar), VARSTRDEF("source"), source);
// Add negate and reset
kvPut(varKv(optionVar), VARSTRDEF("negate"), VARBOOL(cfgOptionNegate(optionId)));
kvPut(varKv(optionVar), VARSTRDEF("reset"), VARBOOL(cfgOptionReset(optionId)));
// Add value if it is set
if (cfgOptionTest(optionId))
{
const Variant *valueVar = NULL;
switch (cfgDefOptionType(cfgOptionDefIdFromId(optionId)))
{
case cfgDefOptTypeBoolean:
case cfgDefOptTypeFloat:
case cfgDefOptTypeInteger:
case cfgDefOptTypePath:
case cfgDefOptTypeSize:
case cfgDefOptTypeString:
{
valueVar = cfgOption(optionId);
break;
}
case cfgDefOptTypeHash:
{
valueVar = varNewKv(kvNew());
const KeyValue *valueKv = cfgOptionKv(optionId);
const VariantList *keyList = kvKeyList(valueKv);
for (unsigned int listIdx = 0; listIdx < varLstSize(keyList); listIdx++)
kvPut(varKv(valueVar), varLstGet(keyList, listIdx), kvGet(valueKv, varLstGet(keyList, listIdx)));
break;
}
case cfgDefOptTypeList:
{
valueVar = varNewKv(kvNew());
const VariantList *valueList = cfgOptionLst(optionId);
for (unsigned int listIdx = 0; listIdx < varLstSize(valueList); listIdx++)
kvPut(varKv(valueVar), varLstGet(valueList, listIdx), BOOL_TRUE_VAR);
break;
}
}
kvPut(varKv(optionVar), VARSTRDEF("value"), valueVar);
}
kvPut(configKv, VARSTRZ(cfgOptionName(optionId)), optionVar);
}
MEM_CONTEXT_PRIOR_BEGIN()
{
result = jsonFromKv(configKv);
}
MEM_CONTEXT_PRIOR_END();
}
MEM_CONTEXT_TEMP_END();
return result;
}

112
libc/xs/config/define.xs
View File

@ -1,112 +0,0 @@
# ----------------------------------------------------------------------------------------------------------------------------------
# Config Definition Perl Exports
# ----------------------------------------------------------------------------------------------------------------------------------
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC
I32
cfgCommandId(commandName)
const char *commandName
CODE:
RETVAL = 0;
ERROR_XS_BEGIN()
{
RETVAL = cfgCommandId(commandName, true);
}
ERROR_XS_END();
OUTPUT:
RETVAL
I32
cfgOptionId(optionName)
const char *optionName
CODE:
RETVAL = 0;
ERROR_XS_BEGIN()
{
RETVAL = cfgOptionId(optionName);
}
ERROR_XS_END();
OUTPUT:
RETVAL
const char *
cfgDefOptionDefault(commandId, optionId)
U32 commandId
U32 optionId
CODE:
RETVAL = NULL;
ERROR_XS_BEGIN()
{
RETVAL = cfgDefOptionDefault(cfgCommandDefIdFromId(commandId), cfgOptionDefIdFromId(optionId));
}
ERROR_XS_END();
OUTPUT:
RETVAL
const char *
cfgDefOptionPrefix(optionId)
U32 optionId
CODE:
RETVAL = NULL;
ERROR_XS_BEGIN()
{
RETVAL = cfgDefOptionPrefix(cfgOptionDefIdFromId(optionId));
}
ERROR_XS_END();
OUTPUT:
RETVAL
bool
cfgDefOptionSecure(optionId)
U32 optionId
CODE:
RETVAL = false;
ERROR_XS_BEGIN()
{
RETVAL = cfgDefOptionSecure(cfgOptionDefIdFromId(optionId));
}
ERROR_XS_END();
OUTPUT:
RETVAL
I32
cfgDefOptionType(optionId);
U32 optionId
CODE:
RETVAL = 0;
ERROR_XS_BEGIN()
{
RETVAL = cfgDefOptionType(cfgOptionDefIdFromId(optionId));
}
ERROR_XS_END();
OUTPUT:
RETVAL
bool
cfgDefOptionValid(commandId, optionId)
U32 commandId
U32 optionId
CODE:
RETVAL = false;
ERROR_XS_BEGIN()
{
RETVAL = cfgDefOptionValid(cfgCommandDefIdFromId(commandId), cfgOptionDefIdFromId(optionId));
}
ERROR_XS_END();
OUTPUT:
RETVAL
U32
cfgOptionTotal()
CODE:
RETVAL = CFG_OPTION_TOTAL;
OUTPUT:
RETVAL

356
libc/xs/storage/storage.xs
View File

@ -1,356 +0,0 @@
# ----------------------------------------------------------------------------------------------------------------------------------
# Storage Exports
# ----------------------------------------------------------------------------------------------------------------------------------
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Storage
####################################################################################################################################
pgBackRest::LibC::Storage
new(class, type, path)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
const String *class = STR_NEW_SV($arg);
const String *type = STR_NEW_SV($arg);
const String *path = STR_NEW_SV($arg);
CODE:
CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::Storage"));
if (strEqZ(type, "<LOCAL>"))
{
MEM_CONTEXT_PRIOR_BEGIN()
{
RETVAL = storagePosixNew(
path == NULL ? STRDEF("/") : path, STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
}
MEM_CONTEXT_PRIOR_END();
}
else if (strEqZ(type, "<REPO>"))
{
CHECK(path == NULL);
RETVAL = (Storage *)storageRepoWrite();
}
else if (strEqZ(type, "<DB>"))
{
CHECK(path == NULL);
MEM_CONTEXT_PRIOR_BEGIN()
{
RETVAL = storagePosixNew(cfgOptionStr(cfgOptPgPath), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
}
MEM_CONTEXT_PRIOR_END();
}
else
THROW_FMT(AssertError, "unexpected storage type '%s'", strPtr(type));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
bucketCreate(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
CODE:
if (strEq(storageType(self), STORAGE_S3_TYPE_STR))
storageS3Request((StorageS3 *)storageDriver(self), HTTP_VERB_PUT_STR, FSLASH_STR, NULL, NULL, true, false);
else
THROW_FMT(AssertError, "unable to create bucket on '%s' storage", strPtr(storageType(self)));
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
bool
copy(self, source, destination)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead source
pgBackRest::LibC::StorageWrite destination
CODE:
RETVAL = storageCopyP(source, destination);
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
SV *
get(self, read)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead read
CODE:
RETVAL = NULL;
Buffer *buffer = storageGetP(read);
if (buffer != NULL)
{
if (bufUsed(buffer) == 0)
RETVAL = newSVpv("", 0);
else
RETVAL = newSVpv((char *)bufPtr(buffer), bufUsed(buffer));
}
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
SV *
info(self, pathExp, ignoreMissing)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *pathExp = STR_NEW_SV($arg);
bool ignoreMissing
CODE:
RETVAL = NULL;
StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = ignoreMissing);
if (info.exists)
{
String *json = storageManifestXsInfo(NULL, &info);
RETVAL = newSVpv((char *)strPtr(json), strSize(json));
}
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
SV *
list(self, pathExp, ignoreMissing, sortAsc, expression)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *pathExp = STR_NEW_SV($arg);
bool ignoreMissing
bool sortAsc
const String *expression = STR_NEW_SV($arg);
CODE:
StringList *fileList = strLstSort(
storageListP(self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false,
.expression = expression), sortAsc ? sortOrderAsc : sortOrderDesc);
const String *fileListJson = jsonFromVar(varNewVarLst(varLstNewStrLst(fileList)));
RETVAL = newSVpv(strPtr(fileListJson), strSize(fileListJson));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
SV *
manifest(self, pathExp, filter=NULL)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *pathExp = STR_NEW_SV($arg);
const String *filter = STR_NEW_SV($arg);
CODE:
StorageManifestXsCallbackData data = {.storage = self, .json = strNew("{"), .pathRoot = pathExp, .filter = filter};
// If a path is specified
StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = true);
if (!info.exists || info.type == storageTypePath)
{
storageInfoListP(
self, data.pathRoot, storageManifestXsCallback, &data,
.errorOnMissing = storageFeature(self, storageFeaturePath) ? true : false);
}
// Else a file is specified
else
{
info.name = strBase(storagePathP(self, pathExp));
strCat(data.json, strPtr(storageManifestXsInfo(NULL, &info)));
}
strCat(data.json, "}");
RETVAL = newSVpv((char *)strPtr(data.json), strSize(data.json));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
SV *
pathGet(self, pathExp)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *pathExp = STR_NEW_SV($arg);
CODE:
String *path = storagePathP(self, pathExp);
RETVAL = newSVpv((char *)strPtr(path), strSize(path));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
pathRemove(self, pathExp, ignoreMissing, recurse)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *pathExp = STR_NEW_SV($arg);
bool ignoreMissing
bool recurse
CODE:
storagePathRemoveP(
self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false, .recurse = recurse);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
UV
put(self, write, buffer)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageWrite write
const Buffer *buffer = BUF_CONST_SV($arg);
CODE:
storagePutP(write, buffer);
RETVAL = buffer ? bufUsed(buffer) : 0;
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
bool
readDrain(self, read)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead read
CODE:
RETVAL = false;
// Read and discard all IO (this is useful for processing filters)
if (ioReadOpen(storageReadIo(read)))
{
Buffer *buffer = bufNew(ioBufferSize());
do
{
ioRead(storageReadIo(read), buffer);
bufUsedZero(buffer);
}
while (!ioReadEof(storageReadIo(read)));
ioReadClose(storageReadIo(read));
RETVAL = true;
}
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
remove(self, fileExp, ignoreMissing)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
const String *fileExp = STR_NEW_SV($arg);
bool ignoreMissing
CODE:
storageRemoveP(self, fileExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
const char *
cipherType(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
CODE:
if (cfgOptionStr(cfgOptRepoCipherType) == NULL || cipherType(cfgOptionStr(cfgOptRepoCipherType)) == cipherTypeNone)
RETVAL = NULL;
else
RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherType));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
const char *
cipherPass(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
CODE:
RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherPass));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
const char *
type(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::Storage self
CODE:
RETVAL = strPtr(storageType(self));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC
####################################################################################################################################
void
storageRepoFree()
CODE:
storageHelperFree();

129
libc/xs/storage/storage.xsh
View File

@ -1,129 +0,0 @@
/***********************************************************************************************************************************
Storage XS Header
***********************************************************************************************************************************/
#include "common/assert.h"
#include "common/compress/gz/compress.h"
#include "common/compress/gz/decompress.h"
#include "common/crypto/cipherBlock.h"
#include "common/io/filter/size.h"
#include "common/memContext.h"
#include "common/type/convert.h"
#include "common/type/json.h"
#include "postgres/interface.h"
#include "storage/helper.h"
#include "storage/s3/storage.intern.h"
#include "storage/storage.intern.h"
typedef Storage *pgBackRest__LibC__Storage;
/***********************************************************************************************************************************
Manifest callback
***********************************************************************************************************************************/
typedef struct StorageManifestXsCallbackData
{
const Storage *storage;
const String *pathRoot;
const String *path;
String *json;
const String *filter;
} StorageManifestXsCallbackData;
String *
storageManifestXsInfo(const String *path, const StorageInfo *info)
{
String *json = strNew("");
if (info->name != NULL)
{
strCatFmt(
json, "%s:", strPtr(jsonFromStr(path == NULL ? info->name : strNewFmt("%s/%s", strPtr(path), strPtr(info->name)))));
}
strCatFmt(json, "{\"group\":%s,\"user\":%s,\"type\":\"", strPtr(jsonFromStr(info->group)), strPtr(jsonFromStr(info->user)));
switch (info->type)
{
case storageTypeFile:
{
strCatFmt(
json, "f\",\"mode\":\"%04o\",\"modification_time\":%" PRId64 ",\"size\":%" PRIu64 "}", info->mode,
(int64_t)info->timeModified, info->size);
break;
}
case storageTypeLink:
{
strCatFmt(json, "l\",\"link_destination\":%s}", strPtr(jsonFromStr(info->linkDestination)));
break;
}
case storageTypePath:
{
strCatFmt(json, "d\",\"mode\":\"%04o\"}", info->mode);
break;
}
case storageTypeSpecial:
{
strCatFmt(json, "s\",\"mode\":\"%04o\"}", info->mode);
break;
}
}
return json;
}
void
storageManifestXsCallback(void *callbackData, const StorageInfo *info)
{
StorageManifestXsCallbackData *data = (StorageManifestXsCallbackData *)callbackData;
if (data->path == NULL || !strEqZ(info->name, "."))
{
if (!strEqZ(info->name, ".") && data->filter && !strEq(data->filter, info->name))
return;
if (strSize(data->json) != 1)
strCat(data->json, ",");
strCat(data->json, strPtr(storageManifestXsInfo(data->path, info)));
if (info->type == storageTypePath)
{
if (!strEqZ(info->name, "."))
{
StorageManifestXsCallbackData dataSub =
{
.storage = data->storage,
.json = data->json,
.pathRoot = data->pathRoot,
.path = data->path == NULL ? info->name : strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)),
};
storageInfoListP(
dataSub.storage, strNewFmt("%s/%s", strPtr(dataSub.pathRoot), strPtr(dataSub.path)), storageManifestXsCallback,
&dataSub);
}
}
}
}
/***********************************************************************************************************************************
Add IO filter
***********************************************************************************************************************************/
void
storageFilterXsAdd(IoFilterGroup *filterGroup, const String *filter, const String *paramJson)
{
VariantList *paramList = paramJson ? jsonToVarLst(paramJson) : NULL;
if (strEqZ(filter, "pgBackRest::Storage::Filter::CipherBlock"))
{
ioFilterGroupAdd(
filterGroup,
cipherBlockNew(
varUInt64Force(varLstGet(paramList, 0)) ? cipherModeEncrypt : cipherModeDecrypt,
cipherType(varStr(varLstGet(paramList, 1))), BUFSTR(varStr(varLstGet(paramList, 2))), NULL));
}
else
THROW_FMT(AssertError, "unable to add invalid filter '%s'", strPtr(filter));
}

126
libc/xs/storage/storageRead.xs
View File

@ -1,126 +0,0 @@
# ----------------------------------------------------------------------------------------------------------------------------------
# Storage Read Exports
# ----------------------------------------------------------------------------------------------------------------------------------
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::StorageRead
####################################################################################################################################
pgBackRest::LibC::StorageRead
new(class, storage, file, ignoreMissing)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
const String *class = STR_NEW_SV($arg);
pgBackRest::LibC::Storage storage
const String *file = STR_NEW_SV($arg);
bool ignoreMissing
CODE:
CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageRead"));
RETVAL = storageReadMove(storageNewReadP(storage, file, .ignoreMissing = ignoreMissing), memContextPrior());
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
filterAdd(self, filter, param)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead self
const String *filter = STR_NEW_SV($arg);
const String *param = STR_NEW_SV($arg);
CODE:
IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(self));
storageFilterXsAdd(filterGroup, filter, param);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
bool
open(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead self
CODE:
RETVAL = ioReadOpen(storageReadIo(self));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
SV *
read(self, bufferSize)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead self
U32 bufferSize
CODE:
RETVAL = NEWSV(0, bufferSize);
SvPOK_only(RETVAL);
Buffer *bufferRead = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL), bufferSize);
ioRead(storageReadIo(self), bufferRead);
SvCUR_set(RETVAL, bufUsed(bufferRead));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
bool
eof(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead self
CODE:
RETVAL = ioReadEof(storageReadIo(self));
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
close(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead self
CODE:
ioReadClose(storageReadIo(self));
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
DESTROY(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageRead self
CODE:
storageReadFree(self);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();

4
libc/xs/storage/storageRead.xsh
View File

@ -1,4 +0,0 @@
/***********************************************************************************************************************************
Storage Read XS Header
***********************************************************************************************************************************/
typedef StorageRead *pgBackRest__LibC__StorageRead;

113
libc/xs/storage/storageWrite.xs
View File

@ -1,113 +0,0 @@
# ----------------------------------------------------------------------------------------------------------------------------------
# Storage Write Exports
# ----------------------------------------------------------------------------------------------------------------------------------
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::StorageWrite
####################################################################################################################################
pgBackRest::LibC::StorageWrite
new(class, storage, file, mode, user, group, timeModified, atomic, pathCreate)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
const String *class = STR_NEW_SV($arg);
pgBackRest::LibC::Storage storage
const String *file = STR_NEW_SV($arg);
U32 mode
const String *user = STR_NEW_SV($arg);
const String *group = STR_NEW_SV($arg);
IV timeModified
bool atomic
bool pathCreate
CODE:
CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageWrite"));
RETVAL = storageWriteMove(
storageNewWriteP(
storage, file, .modeFile = mode, .user = user, .group = group, .timeModified = (time_t)timeModified,
.noCreatePath = storageFeature(storage, storageFeaturePath) ? !pathCreate : false, .noSyncPath = !atomic,
.noAtomic = !atomic),
memContextPrior());
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
filterAdd(self, filter, param)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageWrite self
const String *filter = STR_NEW_SV($arg);
const String *param = STR_NEW_SV($arg);
CODE:
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(self));
storageFilterXsAdd(filterGroup, filter, param);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
open(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageWrite self
CODE:
ioWriteOpen(storageWriteIo(self));
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
UV
write(self, buffer)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageWrite self
const Buffer *buffer = BUF_CONST_SV($arg);
CODE:
ioWrite(storageWriteIo(self), buffer);
RETVAL = bufUsed(buffer);
OUTPUT:
RETVAL
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
close(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageWrite self
CODE:
ioWriteClose(storageWriteIo(self));
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();
####################################################################################################################################
void
DESTROY(self)
PREINIT:
MEM_CONTEXT_XS_TEMP_BEGIN()
{
INPUT:
pgBackRest::LibC::StorageWrite self
CODE:
storageWriteFree(self);
CLEANUP:
}
MEM_CONTEXT_XS_TEMP_END();

4
libc/xs/storage/storageWrite.xsh
View File

@ -1,4 +0,0 @@
/***********************************************************************************************************************************
Storage Write XS Header
***********************************************************************************************************************************/
typedef StorageWrite *pgBackRest__LibC__StorageWrite;

2
test/Vagrantfile vendored
View File

@ -91,7 +91,7 @@ Vagrant.configure(2) do |config|
#---------------------------------------------------------------------------------------------------------------------------
echo 'Build VM Images' && date
rm -rf /backrest/test/.vagrant/docker/*
rm -rf /backrest/test/.vagrant/libc/*
rm -rf /backrest/test/.vagrant/bin/*
rm -rf /backrest/test/.vagrant/package/*
sudo su - vagrant -c '/backrest/test/test.pl --vm-build'

84
test/code-count/file-type.yaml
View File

@ -239,14 +239,6 @@ lib/pgBackRest/InfoCommon.pm:
class: test/harness
type: perl
lib/pgBackRest/LibC.pm:
class: test/harness
type: perl
lib/pgBackRest/LibCAuto.pm:
class: test/harness/auto
type: perl
lib/pgBackRest/Manifest.pm:
class: test/harness
type: perl
@ -279,82 +271,6 @@ lib/pgBackRest/Version.pm:
class: test/harness
type: perl
libc/LibC.h:
class: test/harness
type: c/h
libc/LibC.xs:
class: test/harness
type: xs
libc/Makefile.PL:
class: build
type: perl
libc/build/lib/pgBackRestLibC/Build.pm:
class: build
type: perl
libc/xs/config/config.xs:
class: test/harness
type: xs
libc/xs/config/configTest.xs:
class: test/harness
type: xs
libc/xs/config/configTest.xsh:
class: test/harness
type: c/h
libc/xs/config/define.xs:
class: test/harness
type: xs
libc/xs/crypto/hash.xs:
class: test/harness
type: xs
libc/xs/crypto/hash.xsh:
class: test/harness
type: c/h
libc/xs/postgres/client.xs:
class: test/harness
type: xs
libc/xs/postgres/client.xsh:
class: test/harness
type: c/h
libc/xs/postgres/pageChecksum.xs:
class: test/harness
type: xs
libc/xs/storage/storage.xs:
class: test/harness
type: xs
libc/xs/storage/storage.xsh:
class: test/harness
type: c/h
libc/xs/storage/storageRead.xs:
class: test/harness
type: xs
libc/xs/storage/storageRead.xsh:
class: test/harness
type: c/h
libc/xs/storage/storageWrite.xs:
class: test/harness
type: xs
libc/xs/storage/storageWrite.xsh:
class: test/harness
type: c/h
src/Makefile.in:
class: build
type: make

27
test/lib/pgBackRestTest/Common/BuildTest.pm
View File

@ -113,7 +113,7 @@ sub buildDependencyTree
# Only process non-auto files
if ($strFile =~ /^[A-Za-z0-9\/]+\.(c|h)$/)
{
buildDependencyTreeSub($oStorage, $rhDependencyTree, $strFile, true, undef, ['src', 'libc']);
buildDependencyTreeSub($oStorage, $rhDependencyTree, $strFile, true, undef, ['src']);
}
}
@ -229,8 +229,7 @@ sub buildMakefileObjectCompile
foreach my $strInclude (@{$rhDependencyTree->{$strFile}{include}})
{
$strDepend .=
' ' . ($oStorage->exists("src/${strInclude}") || $strInclude eq BUILD_AUTO_H ? '' : '../libc/') . $strInclude;
$strDepend .= " ${strInclude}";
}
$strMakefile .=
@ -288,27 +287,5 @@ sub buildMakefile
push @EXPORT, qw(buildMakefile);
####################################################################################################################################
# Load the C library and check pointer size
####################################################################################################################################
sub buildLoadLibC
{
# Load the module dynamically
require pgBackRest::LibC;
pgBackRest::LibC->import(qw(:debug));
# Load shared object
require XSLoader;
XSLoader::load('pgBackRest::LibC', '999');
# Do a basic test to make sure it installed correctly
if (libcUvSize() != 8)
{
confess &log(ERROR, 'UVSIZE in test library does not equal 8');
}
}
push @EXPORT, qw(buildLoadLibC);
1;

8
test/lib/pgBackRestTest/Common/CodeCountTest.pm
View File

@ -55,7 +55,6 @@ sub codeCountScan
$strFile =~ '\.eps$' ||
$strFile =~ '\.cache$' ||
$strFile =~ '^doc/site/' ||
$strFile eq 'libc/typemap' ||
$strFile eq 'test/Vagrantfile' ||
$strFile =~ '^test/\.vagrant/' ||
$strFile =~ '^test/certificate/' ||
@ -83,13 +82,12 @@ sub codeCountScan
{
$strClass = 'doc/core';
}
elsif ($strFile =~ '^build/' || $strFile =~ '^libc/build/' || $strFile eq 'libc/Makefile.PL' ||
$strFile eq 'src/Makefile.in' || $strFile eq 'src/configure' || $strFile eq 'src/configure.ac')
elsif ($strFile =~ '^build/' || $strFile eq 'src/Makefile.in' || $strFile eq 'src/configure' ||
$strFile eq 'src/configure.ac')
{
$strClass = 'build';
}
elsif ($strFile =~ '^test/lib/pgBackRestTest/Module/' || $strFile =~ '^test/src/module/' ||
$strFile =~ '^libc/t/')
elsif ($strFile =~ '^test/lib/pgBackRestTest/Module/' || $strFile =~ '^test/src/module/')
{
$strClass = 'test/module';
}

102
test/lib/pgBackRestTest/Common/FileTest.pm
View File

@ -25,7 +25,6 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Storage::Base;
@ -154,7 +153,7 @@ sub forceStorageMode
(
$strOperation,
$oStorage,
$strPathExp,
$strPath,
$strMode,
$bRecurse
) =
@ -162,15 +161,15 @@ sub forceStorageMode
(
__PACKAGE__ . '::forceStorageMode', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'strPath'},
{name => 'strMode'},
{name => 'bRecurse', optional => true, default => false},
);
# Mode commands are ignored on S3
if (!$oStorage->can('type') || $oStorage->type() ne STORAGE_S3)
# Mode commands are ignored on object storage
if ($oStorage->type() ne STORAGE_OBJECT)
{
executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp));
executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} ${strPath}");
}
# Return from function and log return values if any
@ -189,55 +188,48 @@ sub forceStorageMove
(
$strOperation,
$oStorage,
$strSourcePathExp,
$strDestinationPathExp,
$strSourcePath,
$strDestinationPath,
$bRecurse,
) =
logDebugParam
(
__PACKAGE__ . '->forceStorageMove', \@_,
{name => 'oStorage'},
{name => 'strSourcePathExp'},
{name => 'strDestinationPathExp'},
{name => 'strSourcePath'},
{name => 'strDestinationPath'},
{name => 'bRecurse', optional => true, default => true},
);
# If S3 then use storage commands to remove
if ($oStorage->type() eq STORAGE_S3)
# If object storage then use storage commands to remove
if ($oStorage->type() eq STORAGE_OBJECT)
{
if ($bRecurse)
{
my $rhManifest = $oStorage->manifest($strSourcePathExp);
my $rhManifest = $oStorage->manifest($strSourcePath);
foreach my $strName (sort(keys(%{$rhManifest})))
{
if ($rhManifest->{$strName}{type} eq 'f')
{
$oStorage->put(
new pgBackRest::Storage::StorageWrite(
$oStorage,
pgBackRest::LibC::StorageWrite->new(
$oStorage->{oStorageC}, "${strDestinationPathExp}/${strName}", 0, undef, undef, 0, true, false)),
${$oStorage->get(
new pgBackRest::Storage::StorageRead(
$oStorage,
pgBackRest::LibC::StorageRead->new(
$oStorage->{oStorageC}, "${strSourcePathExp}/${strName}", false)))});
$oStorage->remove("${strSourcePathExp}/${strName}");
"${strDestinationPath}/${strName}", ${$oStorage->get("${strSourcePath}/${strName}", {bRaw => true})},
{bRaw => true});
}
}
$oStorage->pathRemove($strSourcePath, {bRecurse => true});
}
else
{
$oStorage->put($strDestinationPathExp, ${$oStorage->get($strSourcePathExp)});
$oStorage->remove($strSourcePathExp);
$oStorage->put($strDestinationPath, ${$oStorage->get($strSourcePath)});
$oStorage->remove($strSourcePath);
}
}
# Else remove using filesystem commands
else
{
executeTest('mv ' . $oStorage->pathGet($strSourcePathExp) . ' ' . $oStorage->pathGet($strDestinationPathExp));
executeTest("mv ${strSourcePath} ${strDestinationPath}");
}
# Return from function and log return values if any
@ -246,41 +238,6 @@ sub forceStorageMove
push(@EXPORT, qw(forceStorageMove));
####################################################################################################################################
# forceStorageOwner - force ownership on a file or path
####################################################################################################################################
sub forceStorageOwner
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorage,
$strPathExp,
$strOwner,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '::forceStorageOwner', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'strOwner'},
{name => 'bRecurse', optional => true, default => false},
);
# Owner commands are ignored on S3
if ($oStorage->type() ne STORAGE_S3)
{
executeTest('chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp));
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push(@EXPORT, qw(forceStorageOwner));
####################################################################################################################################
# forceStorageRemove - force remove a file or path from storage
####################################################################################################################################
@ -291,34 +248,25 @@ sub forceStorageRemove
(
$strOperation,
$oStorage,
$strPathExp,
$strPath,
$bRecurse
) =
logDebugParam
(
__PACKAGE__ . '->forceStorageRemove', \@_,
{name => 'oStorage'},
{name => 'strPathExp'},
{name => 'strPath'},
{name => 'bRecurse', optional => true, default => false},
);
# If S3 then use storage commands to remove
if ($oStorage->type() eq STORAGE_S3)
# If object storage then use storage commands to remove
if ($oStorage->type() eq STORAGE_OBJECT)
{
my $oInfo = $oStorage->info($strPathExp, {bIgnoreMissing => true});
if (defined($oInfo) && $oInfo->{type} eq 'f')
{
$oStorage->remove($strPathExp);
}
else
{
$oStorage->pathRemove($strPathExp, {bRecurse => true});
}
$oStorage->pathRemove($strPath, {bRecurse => true});
}
else
{
executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp));
executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $strPath);
}
# Return from function and log return values if any

5
test/lib/pgBackRestTest/Common/JobTest.pm
View File

@ -221,13 +221,11 @@ sub run
if (!$rhBuildInit->{$self->{oTest}->{&TEST_VM}}{$self->{iVmIdx}})
{
executeTest(
'rsync -rt --delete --exclude=*.o --exclude=test.c --exclude=test.gcno --exclude=LibC.h --exclude=xs' .
'rsync -rt --delete --exclude=*.o --exclude=test.c --exclude=test.gcno ' .
' --exclude=test --exclude=buildflags --exclude=testflags --exclude=harnessflags' .
' --exclude=build.auto.h --exclude=build.auto.h.in --exclude=Makefile --exclude=Makefile.in' .
' --exclude=configure --exclude=configure.ac' .
" $self->{strBackRestBase}/src/ $self->{strGCovPath} && " .
"rsync -t $self->{strBackRestBase}/libc/LibC.h $self->{strGCovPath} && " .
"rsync -rt --delete $self->{strBackRestBase}/libc/xs/ $self->{strGCovPath}/xs && " .
"rsync -rt --delete --exclude=*.o $self->{strBackRestBase}/test/src/ $self->{strGCovPath}/test");
}
@ -794,7 +792,6 @@ sub jobInstallC
my $oVm = vmGet();
my $strBuildPath = "${strBasePath}/test/.vagrant/bin/${strVm}";
my $strBuildLibCPath = "${strBuildPath}/libc";
my $strBuildBinPath = "${strBuildPath}/src";
executeTest(

2
test/lib/pgBackRestTest/Common/ListTest.pm
View File

@ -115,7 +115,7 @@ sub testListGet
# Skip this test if it is integration and vm=none
next if ($strVm eq VM_NONE && $hTest->{&TESTDEF_TYPE} eq TESTDEF_INTEGRATION);
# Skip this test if it is not C and vm=none. Perl tests require libc which is not supported.
# Skip this test if it is not C and vm=none. Perl tests require Docker which is not supported.
next if ($strVm eq VM_NONE && !$hTest->{&TESTDEF_C});
# Skip this test if a container is required and vm=none.

8
test/lib/pgBackRestTest/Common/RunTest.pm
View File

@ -114,7 +114,7 @@ sub process
$self->{iVmId},
$self->{strBasePath},
$self->{strTestPath},
$self->{strBackRestExeC},
$self->{strBackRestExe},
$self->{strBackRestExeHelper},
$self->{strPgBinPath},
$self->{strPgVersion},
@ -136,7 +136,7 @@ sub process
{name => 'iVmId'},
{name => 'strBasePath'},
{name => 'strTestPath'},
{name => 'strBackRestExeC'},
{name => 'strBackRestExe'},
{name => 'strBackRestExeHelper'},
{name => 'strPgBinPath', required => false},
{name => 'strPgVersion', required => false},
@ -159,9 +159,6 @@ sub process
$oStorage = new pgBackRestTest::Common::Storage(
$self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false}));
# Generate backrest exe
$self->{strBackRestExe} = defined($self->{strBackRestExeC}) ? $self->{strBackRestExeC} : $self->{strBackRestExeHelper};
projectBinSet($self->{strBackRestExe});
# Init, run, and end the test(s)
@ -545,6 +542,7 @@ push(@EXPORT, qw(storageTest));
####################################################################################################################################
sub archBits {return vmArchBits(shift->{strVm})}
sub backrestExe {return shift->{strBackRestExe}}
sub backrestExeHelper {return shift->{strBackRestExeHelper}}
sub basePath {return shift->{strBasePath}}
sub dataPath {return shift->basePath() . '/test/data'}
sub doCleanup {return shift->{bCleanup}}

188
test/lib/pgBackRestTest/Common/Storage.pm
View File

@ -69,6 +69,85 @@ sub new
);
}
####################################################################################################################################
# Copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed instead of
# file names.
####################################################################################################################################
sub copy
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xSourceFile,
$xDestinationFile,
$bSourceOpen,
) =
logDebugParam
(
__PACKAGE__ . '->copy', \@_,
{name => 'xSourceFile', required => false},
{name => 'xDestinationFile'},
{name => 'bSourceOpen', optional => true, default => false},
);
# Is source/destination an IO object or a file expression?
my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef;
# Does the source file exist?
my $bResult = false;
# Copy if the source file exists
if (defined($oSourceFileIo))
{
my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile);
# Use C copy if source and destination are C objects
if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite}))
{
$bResult = $self->{oStorageC}->copy(
$oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false;
}
else
{
# Open the source file if it is a C object
$bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true;
if ($bResult)
{
# Open the destination file if it is a C object
if (defined($oDestinationFileIo->{oStorageCWrite}))
{
$oDestinationFileIo->open();
}
# Copy the data
do
{
# Read data
my $tBuffer = '';
$oSourceFileIo->read(\$tBuffer, $self->{lBufferMax});
$oDestinationFileIo->write(\$tBuffer);
}
while (!$oSourceFileIo->eof());
# Close files
$oSourceFileIo->close();
$oDestinationFileIo->close();
}
}
}
return logDebugReturn
(
$strOperation,
{name => 'bResult', value => $bResult, trace => true},
);
}
####################################################################################################################################
# exists - check if file exists
####################################################################################################################################
@ -99,6 +178,62 @@ sub exists
);
}
####################################################################################################################################
# get - reads a buffer from storage all at once
####################################################################################################################################
sub get
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFile,
) =
logDebugParam
(
__PACKAGE__ . '->get', \@_,
{name => 'xFile', required => false, trace => true},
);
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or
# if the repo has a user passphrase defined - else pass undef
my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead($xFile)) : undef;
# Read only if there is something to read from
my $tContent;
my $lSize = 0;
if (defined($oFileIo))
{
my $lSizeRead;
do
{
$lSizeRead = $oFileIo->read(\$tContent, $self->{lBufferMax});
$lSize += $lSizeRead;
}
while ($lSizeRead != 0);
# Close the file
$oFileIo->close();
# If nothing was read then set to undef
if ($lSize == 0)
{
$tContent = undef;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'rtContent', value => defined($oFileIo) ? \$tContent : undef, trace => true},
);
}
####################################################################################################################################
# info - get information for path/file
####################################################################################################################################
@ -595,6 +730,56 @@ sub pathSync
return logDebugReturn($strOperation);
}
####################################################################################################################################
# put - writes a buffer out to storage all at once
####################################################################################################################################
sub put
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$xFile,
$xContent,
) =
logDebugParam
(
__PACKAGE__ . '->put', \@_,
{name => 'xFile', trace => true},
{name => 'xContent', required => false, trace => true},
);
# Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if
# the repo has a user passphrase defined - else pass undef
my $oFileIo = ref($xFile) ? $xFile : $self->openWrite($xFile);
# Determine size of content
my $lSize = defined($xContent) ? length(ref($xContent) ? $$xContent : $xContent) : 0;
# Write only if there is something to write
if ($lSize > 0)
{
$oFileIo->write(ref($xContent) ? $xContent : \$xContent);
}
# Else open the file so a zero length file is created (since file is not opened until first write)
else
{
$oFileIo->open();
}
# Close the file
$oFileIo->close();
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'lSize', value => $lSize, trace => true},
);
}
####################################################################################################################################
# remove - remove path/file
####################################################################################################################################
@ -647,7 +832,6 @@ sub remove
####################################################################################################################################
sub pathBase {shift->{strPathBase}}
sub driver {shift->{oDriver}}
sub cipherType {undef}
sub cipherPassUser {undef}
sub type {shift->{oDriver}->type()}
1;

1
test/lib/pgBackRestTest/Common/StoragePosix.pm
View File

@ -978,5 +978,6 @@ sub remove
sub className {STORAGE_POSIX_DRIVER}
sub tempExtension {shift->{strTempExtension}}
sub tempExtensionSet {my $self = shift; $self->{strTempExtension} = shift}
sub type {STORAGE_POSIX}
1;

2
test/lib/pgBackRestTest/Common/VmTest.pm
View File

@ -509,7 +509,7 @@ sub vmValid
if (!defined($oyVm->{$strVm}))
{
confess &log(ERROR, "no definition for vm '${strVm}'", ERROR_OPTION_INVALID_VALUE);
confess &log(ERROR, "no definition for vm '${strVm}'");
}
}

191
test/lib/pgBackRestTest/Env/ConfigEnvTest.pm
View File

@ -1,191 +0,0 @@
####################################################################################################################################
# ConfigCommonTest.pm - Common code for Config unit tests
####################################################################################################################################
package pgBackRestTest::Env::ConfigEnvTest;
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Getopt::Long qw(GetOptions);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::LibC qw(:test);
use pgBackRest::Version;
use pgBackRestTest::Common::RunTest;
use constant CONFIGENVTEST => 'ConfigEnvTest';
####################################################################################################################################
# Is the option secure?
####################################################################################################################################
sub optionTestSecure
{
my $self = shift;
my $strOption = shift;
return (cfgDefOptionSecure(cfgOptionId($strOption)) ? true : false);
}
sub optionTestSet
{
my $self = shift;
my $iOptionId = shift;
my $strValue = shift;
$self->{&CONFIGENVTEST}{option}{cfgOptionName($iOptionId)} = $strValue;
}
sub optionTestSetBool
{
my $self = shift;
my $iOptionId = shift;
my $bValue = shift;
$self->{&CONFIGENVTEST}{boolean}{cfgOptionName($iOptionId)} = defined($bValue) ? $bValue : true;
}
sub optionTestClear
{
my $self = shift;
my $iOptionId = shift;
delete($self->{&CONFIGENVTEST}{option}{cfgOptionName($iOptionId)});
delete($self->{&CONFIGENVTEST}{boolean}{cfgOptionName($iOptionId)});
}
sub configTestClear
{
my $self = shift;
my $rhConfig = $self->{&CONFIGENVTEST};
delete($self->{&CONFIGENVTEST});
return $rhConfig;
}
sub configTestSet
{
my $self = shift;
my $rhConfig = shift;
$self->{&CONFIGENVTEST} = $rhConfig;
}
####################################################################################################################################
# Write all secure options to a config file
####################################################################################################################################
sub configFileWrite
{
my $self = shift;
my $strConfigFile = shift;
my $rhConfig = shift;
my $strConfig = "[global]\n";
if (defined($rhConfig->{boolean}))
{
foreach my $strOption (sort(keys(%{$rhConfig->{boolean}})))
{
if ($self->optionTestSecure($strOption))
{
$strConfig .= "${strOption}=" . ($rhConfig->{boolean}{$strOption} ? 'y' : 'n') . "\n";
}
}
}
if (defined($rhConfig->{option}))
{
foreach my $strOption (sort(keys(%{$rhConfig->{option}})))
{
if ($self->optionTestSecure($strOption))
{
$strConfig .= "${strOption}=$rhConfig->{option}{$strOption}\n";
}
}
}
storageTest()->put($strConfigFile, $strConfig);
}
####################################################################################################################################
# Write all non-secure options to the command line
####################################################################################################################################
sub commandTestWrite
{
my $self = shift;
my $strCommand = shift;
my $strConfigFile = shift;
my $rhConfig = shift;
my @szyParam = ();
# Add boolean options
if (defined($rhConfig->{boolean}))
{
foreach my $strOption (sort(keys(%{$rhConfig->{boolean}})))
{
if (!$self->optionTestSecure($strOption))
{
if ($rhConfig->{boolean}{$strOption})
{
push(@szyParam, "--${strOption}");
}
else
{
push(@szyParam, "--no-${strOption}");
}
}
}
}
# Add non-boolean options
if (defined($rhConfig->{option}))
{
foreach my $strOption (sort(keys(%{$rhConfig->{option}})))
{
if (!$self->optionTestSecure($strOption))
{
push(@szyParam, "--${strOption}=$rhConfig->{option}{$strOption}");
}
}
}
# Add config file
push(@szyParam, '--' . cfgOptionName(CFGOPT_CONFIG) . "=${strConfigFile}");
# Add command
push(@szyParam, $strCommand);
return @szyParam;
}
####################################################################################################################################
# Load the configuration
####################################################################################################################################
sub configTestLoad
{
my $self = shift;
my $iCommandId = shift;
# A config file is required to store secure options before they can be parsed
my $strConfigFile = $self->testPath() . '/pgbackrest.test.conf';
$self->configFileWrite($strConfigFile, $self->{&CONFIGENVTEST});
my @stryArg = $self->commandTestWrite(cfgCommandName($iCommandId), $strConfigFile, $self->{&CONFIGENVTEST});
my $strConfigJson = cfgParseTest(projectBin(), join('|', @stryArg));
$self->testResult(
sub {configLoad(false, projectBin(), cfgCommandName($iCommandId), \$strConfigJson)},
true, 'config load: ' . join(" ", @stryArg));
}
1;

64
test/lib/pgBackRestTest/Env/ExpireEnvTest.pm
View File

@ -20,10 +20,8 @@ use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
@ -50,6 +48,7 @@ sub new
$self->{oHostBackup},
$self->{strBackRestExe},
$self->{oStorageRepo},
$self->{strPgPath},
$self->{oLogTest},
$self->{oRunTest},
) =
@ -59,6 +58,7 @@ sub new
{name => 'oHostBackup', required => false, trace => true},
{name => 'strBackRestExe', trace => true},
{name => 'oStorageRepo', trace => true},
{name => 'strPgPath', trace => true},
{name => 'oLogTest', required => false, trace => true},
{name => 'oRunTest', required => false, trace => true},
);
@ -124,7 +124,7 @@ sub info
logDebugParam
(
__PACKAGE__ . '->info', \@_,
{name => 'strDbPath', default => cfgOption(CFGOPT_PG_PATH)}
{name => 'strDbPath', default => $self->{strPgPath}}
);
# Open the control file and read system id and versions
@ -168,8 +168,7 @@ sub info
ERROR,
'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} .
' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" .
'HINT: is this version of PostgreSQL supported?',
ERROR_VERSION_NOT_SUPPORTED);
'HINT: is this version of PostgreSQL supported?');
}
# Return from function and log return values if any
@ -210,7 +209,6 @@ sub stanzaSet
my $oStanza = {};
my $oArchiveInfo = {};
my $oBackupInfo = {};
my $bEncrypted = defined($self->{oStorageRepo}->cipherType());
my $iArchiveDbId = 1;
my $iBackupDbId = 1;
@ -218,26 +216,21 @@ sub stanzaSet
if (!$bStanzaUpgrade)
{
$oArchiveInfo =
new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE), false,
{bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_ARCHIVE : undef});
new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath(), false,
{bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef});
$oBackupInfo =
new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP), false,
{bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_MANIFEST : undef});
new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath(), false,
{bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef});
}
# Else get the info data from disk
else
{
$oArchiveInfo =
new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE),
{strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_ARCHIVE : undef});
new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath(),
{strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef});
$oBackupInfo =
new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP),
{strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_MANIFEST : undef});
}
if (cfgOption(CFGOPT_ONLINE))
{
confess &log(ERROR, "this function may not be used for online tests");
new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath(),
{strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef});
}
# Get the database info for the stanza
@ -258,8 +251,8 @@ sub stanzaSet
$oBackupInfo->save();
# Get the archive and directory paths for the stanza
$$oStanza{strArchiveClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE) . '/' . ($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP);
$$oStanza{strArchiveClusterPath} = $self->{oHostBackup}->repoArchivePath($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oHostBackup}->repoBackupPath();
$self->{oStanzaHash}{$strStanza} = $oStanza;
@ -291,14 +284,12 @@ sub stanzaCreate
my $strDbVersionTemp = $strDbVersion;
$strDbVersionTemp =~ s/\.//;
my $strDbPath = cfgOption(CFGOPT_PG_PATH);
# Create the test path for pg_control
storageTest()->pathCreate(($strDbPath . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true});
storageTest()->pathCreate(($self->{strPgPath} . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true});
# Generate pg_control for stanza-create
$self->controlGenerate($strDbPath, $strDbVersion);
executeTest('chmod 600 ' . $strDbPath . '/' . DB_FILE_PGCONTROL);
$self->controlGenerate($self->{strPgPath}, $strDbVersion);
executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
# Create the stanza and set the local stanza object
$self->stanzaSet($strStanza, $strDbVersion, false);
@ -332,11 +323,11 @@ sub stanzaUpgrade
$strDbVersionTemp =~ s/\.//;
# Remove pg_control
storageTest()->remove(cfgOption(CFGOPT_PG_PATH) . '/' . DB_FILE_PGCONTROL);
storageTest()->remove($self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
# Copy pg_control for stanza-upgrade
$self->controlGenerate(cfgOption(CFGOPT_PG_PATH), $strDbVersion);
executeTest('chmod 600 ' . cfgOption(CFGOPT_PG_PATH) . '/' . DB_FILE_PGCONTROL);
$self->controlGenerate($self->{strPgPath}, $strDbVersion);
executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL);
$self->stanzaSet($strStanza, $strDbVersion, true);
@ -393,7 +384,7 @@ sub backupCreate
# Get passphrase (returns undefined if repo not encrypted) to access the manifest
my $strCipherPassManifest =
(new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP)))->cipherPassSub();
(new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath()))->cipherPassSub();
my $strCipherPassBackupSet;
# If repo is encrypted then get passphrase for accessing the backup files from the last manifest if it exists provide one
@ -540,7 +531,7 @@ sub archiveCreate
# Get passphrase (returns undefined if repo not encrypted) to access the archive files
my $strCipherPass =
(new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE)))->cipherPassSub();
(new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath()))->cipherPassSub();
push(my @stryArchive, $strArchive);
@ -604,7 +595,7 @@ sub supplementalLog
join("\n", grep(!/^backup\.info.*$/i, storageRepo()->list("backup/${strStanza}"))));
# Output archive manifest
my $rhManifest = storageRepo()->manifest(STORAGE_REPO_ARCHIVE);
my $rhManifest = storageRepo()->manifest($self->{oHostBackup}->repoArchivePath());
my $strManifest;
my $strPrefix = '';
@ -615,7 +606,7 @@ sub supplementalLog
if ($rhManifest->{$strEntry}->{type} eq 'd')
{
$strEntry = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE) . ($strEntry eq '.' ? '' : "/${strEntry}");
$strEntry = $self->{oHostBackup}->repoArchivePath($strEntry eq '.' ? undef : $strEntry);
# &log(WARN, "DIR $strEntry");
$strManifest .= (defined($strManifest) ? "\n" : '') . "${strEntry}:\n";
@ -670,10 +661,9 @@ sub process
undef($$oStanza{strBackupDescription});
my $strCommand = $self->{strBackRestExe} .
' --' . cfgOptionName(CFGOPT_CONFIG) . '="' . $self->{oHostBackup}->backrestConfig() . '"' .
' --' . cfgOptionName(CFGOPT_STANZA) . '=' . $strStanza .
' --' . cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) . '=' . lc(DETAIL);
my $strCommand =
$self->{strBackRestExe} . ' --config="' . $self->{oHostBackup}->backrestConfig() . '"' . ' --stanza=' . $strStanza .
' --log-level-console=' . lc(DETAIL);
if (defined($iExpireFull))
{

288
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
View File

@ -24,12 +24,11 @@ use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Env::Host::HostS3Test;
@ -50,6 +49,49 @@ use constant HOST_PATH_REPO => 'repo';
use constant HOST_PROTOCOL_TIMEOUT => 10;
push @EXPORT, qw(HOST_PROTOCOL_TIMEOUT);
####################################################################################################################################
# Configuration constants
####################################################################################################################################
use constant CFGDEF_SECTION_GLOBAL => 'global';
push @EXPORT, qw(CFGDEF_SECTION_GLOBAL);
use constant CFGDEF_SECTION_STANZA => 'stanza';
push @EXPORT, qw(CFGDEF_SECTION_STANZA);
use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL);
use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF);
use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr';
push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR);
use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc';
push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
use constant STORAGE_CIFS => 'cifs';
push @EXPORT, qw(STORAGE_CIFS);
use constant STORAGE_S3 => 's3';
push @EXPORT, qw(STORAGE_S3);
use constant CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_DEFAULT);
use constant CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_IMMEDIATE);
use constant CFGOPTVAL_RESTORE_TYPE_NAME => 'name';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_NAME);
use constant CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_PRESERVE);
use constant CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_STANDBY);
use constant CFGOPTVAL_RESTORE_TYPE_TIME => 'time';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_TIME);
use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid';
push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_XID);
use constant NONE => 'none';
push @EXPORT, qw(NONE);
use constant GZ => 'gz';
push @EXPORT, qw(GZ);
####################################################################################################################################
# new
####################################################################################################################################
@ -251,8 +293,8 @@ sub backupEnd
{
if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink())
{
my $hTablespaceManifest = storageRepo()->manifest(
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
my $hTablespaceManifest = storageTest()->manifest(
$self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC));
# Remove . and ..
delete($hTablespaceManifest->{'.'});
@ -305,19 +347,17 @@ sub backupEnd
}
# Else there should not be a tablespace directory at all. This is only valid for storage that supports links.
elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) &&
storageTest()->pathExists(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)))
storageTest()->pathExists($self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)))
{
confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
}
}
# Check that latest link exists unless repo links are disabled
my $strLatestLink = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST);
my $strLatestLink = $self->repoBackupPath(LINK_LATEST);
my $bLatestLinkExists = storageRepo()->exists($strLatestLink);
if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq CFGOPTVAL_REPO_TYPE_POSIX) && $self->hasLink())
if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq STORAGE_POSIX) && $self->hasLink())
{
my $strLatestLinkDestination = readlink($strLatestLink);
@ -367,14 +407,12 @@ sub backupEnd
if ($self->synthetic() && $bManifestCompare)
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), undef,
$self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), undef,
${storageRepo()->get(
storageRepo()->openRead(
STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST,
{strCipherPass => $self->cipherPassManifest()}))});
$self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}))});
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
$self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo->get($self->repoBackupPath(FILE_BACKUP_INFO))});
}
}
@ -446,8 +484,7 @@ sub backupCompare
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup;
my $oActualManifest = new pgBackRest::Manifest(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST),
{strCipherPass => $self->cipherPassManifest()});
$self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()});
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} =
$oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START);
@ -478,7 +515,7 @@ sub backupCompare
# Determine repo size if compression or encryption is enabled
my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE};
if ($strCompressType ne CFGOPTVAL_COMPRESS_TYPE_NONE ||
if ($strCompressType ne NONE ||
(defined($oExpectedManifest->{&INI_SECTION_CIPHER}) &&
defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS})))
{
@ -486,9 +523,9 @@ sub backupCompare
my $lRepoSize =
$oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
$oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
(storageRepo()->info(STORAGE_REPO_BACKUP .
"/${strBackup}/${strFileKey}" .
($strCompressType eq CFGOPTVAL_COMPRESS_TYPE_NONE ? '' : ".${strCompressType}")))->{size};
(storageRepo()->info(
$self->repoBackupPath("${strBackup}/${strFileKey}") .
($strCompressType eq NONE ? '' : ".${strCompressType}")))->{size};
if (defined($lRepoSize) &&
$lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
@ -613,7 +650,7 @@ sub backupLast
my $self = shift;
my @stryBackup = storageRepo()->list(
STORAGE_REPO_BACKUP, {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'});
$self->repoBackupPath(), {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'});
if (!defined($stryBackup[0]))
{
@ -780,13 +817,11 @@ sub stanzaCreate
if (defined($self->{oLogTest}) && $self->synthetic())
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
$self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo()->get($self->repoBackupPath(FILE_BACKUP_INFO))});
}
# Get the passphrase for accessing the manifest file
$self->{strCipherPassManifest} =
(new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza())))->cipherPassSub();
$self->{strCipherPassManifest} = (new pgBackRest::Backup::Info($self->repoBackupPath()))->cipherPassSub();
}
if (storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE))
@ -795,13 +830,12 @@ sub stanzaCreate
if (defined($self->{oLogTest}) && $self->synthetic())
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
$self->repoArchivePath(ARCHIVE_INFO_FILE), undef, ${storageRepo()->get($self->repoArchivePath(ARCHIVE_INFO_FILE))});
}
# Get the passphrase for accessing the archived files
$self->{strCipherPassArchive} =
(new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza())))->cipherPassSub();
(new pgBackRest::Archive::Info($self->repoArchivePath()))->cipherPassSub();
}
# Return from function and log return values if any
@ -848,16 +882,14 @@ sub stanzaUpgrade
storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO))
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
$self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo()->get($self->repoBackupPath(FILE_BACKUP_INFO))});
}
if (defined($self->{oLogTest}) && $self->synthetic() &&
storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE))
{
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
$self->repoArchivePath(ARCHIVE_INFO_FILE), undef, ${storageRepo()->get($self->repoArchivePath(ARCHIVE_INFO_FILE))});
}
# Return from function and log return values if any
@ -1009,65 +1041,64 @@ sub configCreate
# General options
# ------------------------------------------------------------------------------------------------------------------------------
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE)} = lc(DETAIL);
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_FILE)} = testRunGet()->logLevelTestFile();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_STDERR)} = lc(OFF);
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_SUBPROCESS)} =
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-console'} = lc(DETAIL);
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-file'} = testRunGet()->logLevelTestFile();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-stderr'} = lc(OFF);
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-subprocess'} =
testRunGet()->logLevelTestFile() eq lc(OFF) ? 'n' : 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_TIMESTAMP)} = 'n';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_BUFFER_SIZE)} = '64k';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-timestamp'} = 'n';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'buffer-size'} = '64k';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_PATH)} = $self->logPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOCK_PATH)} = $self->lockPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_PROTOCOL_TIMEOUT)} = 60;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_DB_TIMEOUT)} = 45;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'protocol-timeout'} = 60;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'db-timeout'} = 45;
# Set to make sure that changing the default works and to speed compression for testing
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_LEVEL)} = 3;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level'} = 3;
# Only set network compress level if there is more than one host
if ($oHostBackup != $oHostDbMaster)
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_LEVEL_NETWORK)} = 1;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level-network'} = 1;
}
if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne CFGOPTVAL_COMPRESS_TYPE_GZ)
if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne 'gz')
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_TYPE)} = $oParam->{strCompressType};
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-type'} = $oParam->{strCompressType};
}
if ($self->isHostBackup())
{
if ($self->repoEncrypt())
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_CIPHER_TYPE)} =
CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_CIPHER_PASS)} = 'x';
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-type'} = CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-pass'} = 'x';
}
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_PATH)} = $self->repoPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath();
# S3 settings
if ($oParam->{bS3})
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_TYPE)} = CFGOPTVAL_REPO_TYPE_S3;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_KEY)} = HOST_S3_ACCESS_KEY;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_KEY_SECRET)} = HOST_S3_ACCESS_SECRET_KEY;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_BUCKET)} = HOST_S3_BUCKET;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_ENDPOINT)} = HOST_S3_ENDPOINT;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_REGION)} = HOST_S3_REGION;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = STORAGE_S3;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key'} = HOST_S3_ACCESS_KEY;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key-secret'} = HOST_S3_ACCESS_SECRET_KEY;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-bucket'} = HOST_S3_BUCKET;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-endpoint'} = HOST_S3_ENDPOINT;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-region'} = HOST_S3_REGION;
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'} = 'n';
}
if (defined($$oParam{bHardlink}) && $$oParam{bHardlink})
{
$self->{bHardLink} = true;
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_REPO_HARDLINK)} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'repo1-s3-hardlink'} = 'y';
}
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_ARCHIVE_COPY)} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_START_FAST)} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'archive-copy'} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y';
}
# Host specific options
@ -1087,53 +1118,44 @@ sub configCreate
if ($self->nameTest(HOST_BACKUP))
{
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST)} = $oHostDb1->nameGet();
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_USER)} = $oHostDb1->userGet();
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_CMD)} = $oHostDb1->backrestExe();
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_CONFIG)} = $oHostDb1->backrestConfig();
$oParamHash{$strStanza}{'pg1-host'} = $oHostDb1->nameGet();
$oParamHash{$strStanza}{'pg1-host-user'} = $oHostDb1->userGet();
$oParamHash{$strStanza}{'pg1-host-cmd'} = $oHostDb1->backrestExe();
$oParamHash{$strStanza}{'pg1-host-config'} = $oHostDb1->backrestConfig();
# Port can't be configured for a synthetic host
if (!$self->synthetic())
{
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PORT)} = $oHostDb1->pgPort();
$oParamHash{$strStanza}{'pg1-port'} = $oHostDb1->pgPort();
}
}
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $oHostDb1->dbBasePath();
$oParamHash{$strStanza}{'pg1-path'} = $oHostDb1->dbBasePath();
if (defined($oHostDb2))
{
# Add an invalid replica to simulate more than one replica. A warning should be thrown when a stanza is created and a
# valid replica should be chosen.
my $iInvalidReplica = 2;
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iInvalidReplica))} = BOGUS;
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iInvalidReplica))} =
$oHostDb2->userGet();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iInvalidReplica))} =
$oHostDb2->backrestExe();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iInvalidReplica))} =
$oHostDb2->backrestConfig();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iInvalidReplica))} =
$oHostDb2->dbBasePath();
$oParamHash{$strStanza}{"pg2-host"} = BOGUS;
$oParamHash{$strStanza}{"pg2-host-user"} = $oHostDb2->userGet();
$oParamHash{$strStanza}{"pg2-host-cmd"} = $oHostDb2->backrestExe();
$oParamHash{$strStanza}{"pg2-host-config"} = $oHostDb2->backrestConfig();
$oParamHash{$strStanza}{"pg2-path"} = $oHostDb2->dbBasePath();
# Set a flag so we know there's a bogus host
$self->{bBogusHost} = true;
# Set a valid replica to the last possible index to ensure skipping indexes does not make a difference.
my $iValidReplica = cfgOptionIndexTotal(CFGOPT_PG_PATH);
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iValidReplica))} = $oHostDb2->nameGet();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iValidReplica))} =
$oHostDb2->userGet();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iValidReplica))} =
$oHostDb2->backrestExe();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iValidReplica))} =
$oHostDb2->backrestConfig();
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iValidReplica))} = $oHostDb2->dbBasePath();
# Set a valid replica to a higher index to ensure skipping indexes does not make a difference
$oParamHash{$strStanza}{"pg8-host"} = $oHostDb2->nameGet();
$oParamHash{$strStanza}{"pg8-host-user"} = $oHostDb2->userGet();
$oParamHash{$strStanza}{"pg8-host-cmd"} = $oHostDb2->backrestExe();
$oParamHash{$strStanza}{"pg8-host-config"} = $oHostDb2->backrestConfig();
$oParamHash{$strStanza}{"pg8-path"} = $oHostDb2->dbBasePath();
# Only test explicit ports on the backup server. This is so locally configured ports are also tested.
if (!$self->synthetic() && $self->nameTest(HOST_BACKUP))
{
$oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iValidReplica))} = $oHostDb2->pgPort();
$oParamHash{$strStanza}{"pg8-port"} = $oHostDb2->pgPort();
}
}
}
@ -1141,32 +1163,31 @@ sub configCreate
# If this is a database host
if ($self->isHostDb())
{
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $self->dbBasePath();
$oParamHash{$strStanza}{'pg1-path'} = $self->dbBasePath();
if (!$self->synthetic())
{
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_SOCKET_PATH)} = $self->pgSocketPath();
$oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PORT)} = $self->pgPort();
$oParamHash{$strStanza}{'pg1-socket-path'} = $self->pgSocketPath();
$oParamHash{$strStanza}{'pg1-port'} = $self->pgPort();
}
if ($bArchiveAsync)
{
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':' .
cfgCommandName(CFGCMD_ARCHIVE_PUSH)}{cfgOptionName(CFGOPT_ARCHIVE_ASYNC)} = 'y';
$oParamHash{&CFGDEF_SECTION_GLOBAL . ':archive-push'}{'archive-async'} = 'y';
}
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_SPOOL_PATH)} = $self->spoolPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'spool-path'} = $self->spoolPath();
# If the the backup host is remote
if (!$self->isHostBackup())
{
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST)} = $oHostBackup->nameGet();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_USER)} = $oHostBackup->userGet();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_CMD)} = $oHostBackup->backrestExe();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_CONFIG)} = $oHostBackup->backrestConfig();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host'} = $oHostBackup->nameGet();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-user'} = $oHostBackup->userGet();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cmd'} = $oHostBackup->backrestExe();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-config'} = $oHostBackup->backrestConfig();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_PATH)} = $self->logPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOCK_PATH)} = $self->lockPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath();
$oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath();
}
}
@ -1244,7 +1265,7 @@ sub manifestMunge
{name => 'bCache', default => true},
);
$self->infoMunge(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true);
$self->infoMunge($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true);
# Return from function and log return values if any
return logDebugReturn($strOperation);
@ -1271,7 +1292,7 @@ sub manifestRestore
{name => 'bSave', default => true},
);
$self->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $bSave);
$self->infoRestore($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $bSave);
# Return from function and log return values if any
return logDebugReturn($strOperation);
@ -1309,15 +1330,16 @@ sub infoMunge
# If the original file content does not exist then load it
if (!defined($self->{hInfoFile}{$strFileName}))
{
$self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini($strFileName, {oStorage => storageRepo(),
strCipherPass => !$bManifest ? storageRepo()->cipherPassUser() : $self->cipherPassManifest()});
$self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini(
storageRepo(), $strFileName,
{strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()});
}
# Make a copy of the original file contents
my $oMungeIni = new pgBackRest::Common::Ini(
$strFileName,
{bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), oStorage => storageRepo(),
strCipherPass => !$bManifest ? storageRepo()->cipherPassUser() : $self->cipherPassManifest()});
storageRepo(), $strFileName,
{bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}),
strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()});
# Load params
foreach my $strSection (keys(%{$hParam}))
@ -1438,7 +1460,7 @@ sub configRecovery
if (@stryRecoveryOption)
{
$oConfig->{$strStanza}{cfgOptionName(CFGOPT_RECOVERY_OPTION)} = \@stryRecoveryOption;
$oConfig->{$strStanza}{'recovery-option'} = \@stryRecoveryOption;
}
# Save db config file
@ -1472,7 +1494,7 @@ sub configRemap
}
# Rewrite recovery section
delete($oConfig->{"${strStanza}:restore"}{cfgOptionName(CFGOPT_TABLESPACE_MAP)});
delete($oConfig->{"${strStanza}:restore"}{'tablespace-map'});
my @stryTablespaceMap;
foreach my $strRemap (sort(keys(%$oRemapHashRef)))
@ -1481,13 +1503,13 @@ sub configRemap
if ($strRemap eq MANIFEST_TARGET_PGDATA)
{
$oConfig->{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $strRemapPath;
$oConfig->{$strStanza}{'pg1-path'} = $strRemapPath;
${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} = $strRemapPath;
if (defined($oHostBackup))
{
$oRemoteConfig->{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $strRemapPath;
$oRemoteConfig->{$strStanza}{'pg1-path'} = $strRemapPath;
}
}
else
@ -1502,7 +1524,7 @@ sub configRemap
if (@stryTablespaceMap)
{
$oConfig->{"${strStanza}:restore"}{cfgOptionName(CFGOPT_TABLESPACE_MAP)} = \@stryTablespaceMap;
$oConfig->{"${strStanza}:restore"}{'tablespace-map'} = \@stryTablespaceMap;
}
# Save db config file
@ -1577,14 +1599,13 @@ sub restore
$strComment = 'restore' .
($bDelta ? ' delta' : '') .
($bForce ? ', force' : '') .
($strBackup ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? ", backup '${strBackup}'" : '') .
($strBackup ne 'latest' ? ", backup '${strBackup}'" : '') .
# This does not output 'default' for synthetic tests to make expect logs match up (may change later)
($strType ? ", type '${strType}'" : (defined($rhExpectedManifest) ? '' : ", type 'default'")) .
($strTarget ? ", target '${strTarget}'" : '') .
($strTargetTimeline ? ", timeline '${strTargetTimeline}'" : '') .
($bTargetExclusive ? ', exclusive' : '') .
(defined($strTargetAction) && $strTargetAction ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_TARGET_ACTION)
? ', ' . cfgOptionName(CFGOPT_TARGET_ACTION) . "=${strTargetAction}" : '') .
(defined($strTargetAction) && $strTargetAction ne 'pause' ? ", target-action=${strTargetAction}" : '') .
(defined($rhRemapHash) ? ', remap' : '') .
(defined($iExpectedExitStatus) ? ", expect exit ${iExpectedExitStatus}" : '') .
(defined($strComment) ? " - ${strComment}" : '') .
@ -1602,7 +1623,7 @@ sub restore
# - which should be the backup passed as strBackupExpected. If it is not defined, then set it based on the strBackup passed.
if (!defined($strBackupExpected))
{
$strBackupExpected = $strBackup eq cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? $oHostBackup->backupLast() :
$strBackupExpected = $strBackup eq 'latest' ? $oHostBackup->backupLast() :
$strBackup;
}
@ -1610,9 +1631,7 @@ sub restore
{
# Load the manifest from the backup expected to be chosen/processed by restore
my $oExpectedManifest = new pgBackRest::Manifest(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} . $strBackupExpected. qw{/} .
FILE_MANIFEST),
$self->repoBackupPath($strBackupExpected . qw{/} . FILE_MANIFEST),
{strCipherPass => $oHostBackup->cipherPassManifest()});
$rhExpectedManifest = $oExpectedManifest->{oContent};
@ -1672,7 +1691,7 @@ sub restore
' --config=' . $self->backrestConfig() .
($bDelta ? ' --delta' : '') .
($bForce ? ' --force' : '') .
($strBackup ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? " --set=${strBackup}" : '') .
($strBackup ne 'latest' ? " --set=${strBackup}" : '') .
(defined($strOptionalParam) ? " ${strOptionalParam} " : '') .
(defined($strType) && $strType ne CFGOPTVAL_RESTORE_TYPE_DEFAULT ? " --type=${strType}" : '') .
(defined($strTarget) ? " --target=\"${strTarget}\"" : '') .
@ -1680,8 +1699,7 @@ sub restore
($bTargetExclusive ? ' --target-exclusive' : '') .
(defined($strLinkMap) ? $strLinkMap : '') .
($self->synthetic() ? '' : ' --link-all') .
(defined($strTargetAction) && $strTargetAction ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_TARGET_ACTION)
? ' --' . cfgOptionName(CFGOPT_TARGET_ACTION) . "=${strTargetAction}" : '') .
(defined($strTargetAction) && $strTargetAction ne 'pause' ? " --target-action=${strTargetAction}" : '') .
' --stanza=' . $self->stanza() . ' restore',
{strComment => $strComment, iExpectedExitStatus => $iExpectedExitStatus, oLogTest => $self->{oLogTest},
bLogOutput => $self->synthetic()},
@ -1723,9 +1741,8 @@ sub restoreCompare
{
my $oExpectedManifest =
new pgBackRest::Manifest(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} . ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) .
'/'. FILE_MANIFEST),
$self->repoBackupPath(
($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST),
{strCipherPass => $oHostBackup->cipherPassManifest()});
# Get the --delta option from the backup manifest so the actual manifest can be built the same way for comparison
@ -1734,9 +1751,8 @@ sub restoreCompare
$oLastManifest =
new pgBackRest::Manifest(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} .
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST),
$self->repoBackupPath(
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST),
{strCipherPass => $oHostBackup->cipherPassManifest()});
}
@ -2027,6 +2043,20 @@ sub restoreCompare
storageTest()->remove("${strTestPath}/actual.manifest");
}
####################################################################################################################################
# Get repo backup/archive path
####################################################################################################################################
sub repoSubPath
{
my $self = shift;
my $strSubPath = shift;
my $strPath = shift;
return
($self->{strRepoPath} eq '/' ? '' : $self->{strRepoPath}) . "/${strSubPath}/" . $self->stanza() .
(defined($strPath) ? "/${strPath}" : '');
}
####################################################################################################################################
# Getters
####################################################################################################################################
@ -2036,13 +2066,15 @@ sub backrestExe {return testRunGet()->backrestExe()}
sub bogusHost {return shift->{bBogusHost}}
sub hardLink {return shift->{bHardLink}}
sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)}
sub isFS {storageRepo()->type() ne STORAGE_S3}
sub isFS {storageRepo()->type() ne STORAGE_OBJECT}
sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()}
sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER}
sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY}
sub isHostDb {my $self = shift; return $self->isHostDbMaster() || $self->isHostDbStandby()}
sub lockPath {return shift->{strLockPath}}
sub logPath {return shift->{strLogPath}}
sub repoArchivePath {return shift->repoSubPath('archive', shift)}
sub repoBackupPath {return shift->repoSubPath('backup', shift)}
sub repoPath {return shift->{strRepoPath}}
sub repoEncrypt {return shift->{bRepoEncrypt}}
sub stanza {return testRunGet()->stanza()}

2
test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm
View File

@ -17,7 +17,7 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use pgBackRest::Common::Log;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;

3
test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm
View File

@ -21,10 +21,9 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;

2
test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm
View File

@ -23,7 +23,7 @@ use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;

2
test/lib/pgBackRestTest/Env/Host/HostDbTest.pm
View File

@ -22,7 +22,7 @@ use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;

3
test/lib/pgBackRestTest/Env/Host/HostS3Test.pm
View File

@ -22,9 +22,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBaseTest;

47
test/lib/pgBackRestTest/Env/HostEnvTest.pm
View File

@ -2,7 +2,7 @@
# FullCommonTest.pm - Common code for backup tests
####################################################################################################################################
package pgBackRestTest::Env::HostEnvTest;
use parent 'pgBackRestTest::Env::ConfigEnvTest';
use parent 'pgBackRestTest::Common::RunTest';
####################################################################################################################################
# Perl includes
@ -18,9 +18,9 @@ use Storable qw(dclone);
use pgBackRest::Archive::Common;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
@ -43,11 +43,6 @@ use constant ENCRYPTION_KEY_MANIFEST => 'manifest';
use constant ENCRYPTION_KEY_BACKUPSET => 'backupset';
push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET);
use constant NONE => CFGOPTVAL_COMPRESS_TYPE_NONE;
push @EXPORT, qw(NONE);
use constant GZ => CFGOPTVAL_COMPRESS_TYPE_GZ;
push @EXPORT, qw(GZ);
####################################################################################################################################
# setup
####################################################################################################################################
@ -151,6 +146,13 @@ sub setup
$oHostBackup = $strBackupDestination eq HOST_DB_MASTER ? $oHostDbMaster : $oHostDbStandby;
}
storageRepoCommandSet(
$self->backrestExeHelper() .
' --config=' . $oHostBackup->backrestConfig() . ' --stanza=' . $self->stanza() . ' --log-level-console=off' .
' --log-level-stderr=error' .
($oConfigParam->{bS3} ? ' --no-repo1-s3-verify-tls --repo1-s3-host=' . $oHostS3->ipGet() : ''),
$oConfigParam->{bS3} ? STORAGE_OBJECT : STORAGE_POSIX);
# Create db-standby config
if (defined($oHostDbStandby))
{
@ -161,37 +163,10 @@ sub setup
bArchiveAsync => $$oConfigParam{bArchiveAsync}});
}
# Set options needed for storage helper
$self->optionTestSet(CFGOPT_PG_PATH, $oHostDbMaster->dbBasePath());
$self->optionTestSet(CFGOPT_REPO_PATH, $oHostBackup->repoPath());
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
# Configure the repo to be encrypted if required
if ($bRepoEncrypt)
{
$self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
$self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, 'x');
}
# Set S3 options
if (defined($oHostS3))
{
$self->optionTestSet(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3);
$self->optionTestSet(CFGOPT_REPO_S3_KEY, HOST_S3_ACCESS_KEY);
$self->optionTestSet(CFGOPT_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY);
$self->optionTestSet(CFGOPT_REPO_S3_BUCKET, HOST_S3_BUCKET);
$self->optionTestSet(CFGOPT_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT);
$self->optionTestSet(CFGOPT_REPO_S3_REGION, HOST_S3_REGION);
$self->optionTestSet(CFGOPT_REPO_S3_HOST, $oHostS3->ipGet());
$self->optionTestSetBool(CFGOPT_REPO_S3_VERIFY_TLS, false);
}
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Create S3 bucket
if (defined($oHostS3))
{
storageRepo()->{oStorageC}->bucketCreate();
storageRepo()->create();
}
return $oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3;

83
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
View File

@ -21,10 +21,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
@ -93,12 +91,12 @@ sub run
# without slowing down the other tests too much.
if ($bS3)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
# Reduce log level to warn because parallel tests do not create deterministic logs
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(WARN)}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(WARN)}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(WARN)}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(WARN)}});
}
# Get base time
@ -115,7 +113,7 @@ sub run
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 16384;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE} = JSON::PP::true;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::false;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = CFGOPTVAL_COMPRESS_TYPE_NONE;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = NONE;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL} = 3;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK} = $bRemote ? 1 : 3;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::false;
@ -306,7 +304,7 @@ sub run
my $strTestPoint;
# Create the archive info file
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-online'});
# Create a link to postgresql.conf
storageTest()->pathCreate($oHostDbMaster->dbPath() . '/pg_config', {strMode => '0700', bCreateParent => true});
@ -343,14 +341,13 @@ sub run
{oExpectedManifest => \%oManifest,
strOptionalParam => $strOptionalParam .
# Pass ssh path to make sure it is used
($bRemote ? ' --' . cfgOptionName(CFGOPT_CMD_SSH) . '=/usr/bin/ssh' : '') .
($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') .
# Pass bogus ssh port to make sure it is passed through the protocol layer (it won't be used)
($bRemote ? ' --' . cfgOptionName(CFGOPT_PG_PORT) . '=9999' : '') .
($bRemote ? ' --pg1-port=9999' : '') .
# Pass bogus socket path to make sure it is passed through the protocol layer (it won't be used)
($bRemote ? ' --' . cfgOptionName(CFGOPT_PG_SOCKET_PATH) . '=/test_socket_path' : '') .
' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=16384 --' . cfgOptionName(CFGOPT_CHECKSUM_PAGE) .
' --' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1',
strRepoType => $bS3 ? undef : CFGOPTVAL_REPO_TYPE_CIFS, strTest => $strTestPoint, fTestDelay => 0});
($bRemote ? ' --pg1-socket-path=/test_socket_path' : '') .
' --buffer-size=16384 --checksum-page --process-max=1',
strRepoType => $bS3 ? undef : STORAGE_CIFS, strTest => $strTestPoint, fTestDelay => 0});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 65536;
@ -409,13 +406,13 @@ sub run
# Resume by copying the valid full backup over the last aborted full backup if it exists, or by creating a new path
my $strResumeBackup = (storageRepo()->list(
STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))[0];
$oHostBackup->repoBackupPath(), {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))[0];
my $strResumeLabel = $strResumeBackup ne $strFullBackup ?
$strResumeBackup : backupLabel(storageRepo(), $strType, undef, time());
my $strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . '/' . $strResumeLabel);
$strResumeBackup : backupLabel(storageRepo(), $oHostBackup->repoBackupPath(), $strType, undef, time());
my $strResumePath = $oHostBackup->repoBackupPath($strResumeLabel);
forceStorageRemove(storageRepo(), $strResumePath, {bRecurse => true});
forceStorageMove(storageRepo(), 'backup/' . $self->stanza() . "/${strFullBackup}", $strResumePath);
forceStorageMove(storageRepo(), $oHostBackup->repoBackupPath($strFullBackup), $strResumePath);
# Set ownership on base directory to bogus values
if (!$bRemote)
@ -464,7 +461,7 @@ sub run
# Create files to be excluded with the --exclude option
$oHostBackup->configUpdate(
{(CFGDEF_SECTION_GLOBAL . ':backup') =>
{cfgOptionName(CFGOPT_EXCLUDE) => ['postgresql.auto.conf', 'pg_log/', 'pg_log2', 'apipe']}});
{'exclude' => ['postgresql.auto.conf', 'pg_log/', 'pg_log2', 'apipe']}});
$oHostDbMaster->dbLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.auto.conf',
'../pg_config/postgresql.conf', true);
$oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_log');
@ -476,7 +473,7 @@ sub run
$strFullBackup = $oHostBackup->backup(
$strType, 'resume',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--force --' . cfgOptionName(CFGOPT_CHECKSUM_PAGE) . ($bDeltaBackup ? ' --delta' : '')});
strOptionalParam => '--force --checksum-page' . ($bDeltaBackup ? ' --delta' : '')});
# Remove postmaster.pid so restore will succeed (the rest will be cleaned up by the delta)
storageTest->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
@ -523,7 +520,7 @@ sub run
strOptionalParam => ' --link-all' . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '')});
# Remove excludes now that they just create noise in the log
$oHostBackup->configUpdate({(CFGDEF_SECTION_GLOBAL . ':backup') => {cfgOptionName(CFGOPT_EXCLUDE) => []}});
$oHostBackup->configUpdate({(CFGDEF_SECTION_GLOBAL . ':backup') => {'exclude' => []}});
# Run again to fix permissions
if (!$bRemote)
@ -624,11 +621,11 @@ sub run
$strType = CFGOPTVAL_BACKUP_TYPE_INCR;
# Create resumable backup from last backup
$strResumeLabel = backupLabel(storageRepo(), $strType, substr($strBackup, 0, 16), time());
$strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . '/' . $strResumeLabel);
$strResumeLabel = backupLabel(storageRepo(), $oHostBackup->repoBackupPath(), $strType, substr($strBackup, 0, 16), time());
$strResumePath = $oHostBackup->repoBackupPath($strResumeLabel);
forceStorageRemove(storageRepo(), $strResumePath);
forceStorageMove(storageRepo(), 'backup/' . $self->stanza() . "/${strBackup}", $strResumePath);
forceStorageMove(storageRepo(), $oHostBackup->repoBackupPath($strBackup), $strResumePath);
# Munge manifest so the resumed file in the repo appears to be bad
if ($bEncrypt || $bRemote)
@ -698,7 +695,7 @@ sub run
$strBackup = $oHostBackup->backup(
$strType, 'resume and add tablespace 2',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')});
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@ -719,9 +716,7 @@ sub run
$strBackup = $oHostBackup->backup(
$strType, 'drop tablespace 11',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' .
($bDeltaBackup ? ' --delta' : '')});
{oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@ -776,7 +771,7 @@ sub run
$strBackup = $oHostBackup->backup(
$strType, 'add files and remove tablespace 2',
{oExpectedManifest => \%oManifest, strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1'});
{oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1'});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@ -801,7 +796,7 @@ sub run
$strBackup = $oHostBackup->backup(
$strType, 'updates since last full', {oExpectedManifest => \%oManifest,
strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')});
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@ -814,17 +809,17 @@ sub run
# Enable compression to ensure a warning is raised (reset when gz to avoid log churn since it is the default)
if ($strCompressType eq GZ)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => undef}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'compress-type' => undef}});
}
else
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => $strCompressType}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'compress-type' => $strCompressType}});
}
# Enable hardlinks (except for s3) to ensure a warning is raised
if (!$bS3)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_REPO_HARDLINK) => 'y'}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'repo1-hardlink' => 'y'}});
}
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1;
@ -840,7 +835,7 @@ sub run
$oHostBackup->backup(
$strType, 'remove files',
{oExpectedManifest => \%oManifest,
strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')});
strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')});
$oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1;
@ -889,8 +884,7 @@ sub run
$strFullBackup, {&MANIFEST_SECTION_BACKUP_OPTION => {&MANIFEST_KEY_CHECKSUM_PAGE => undef}}, false);
$strBackup = $oHostBackup->backup(
$strType, 'add file',
{oExpectedManifest => \%oManifest, strOptionalParam => '--' . cfgOptionName(CFGOPT_CHECKSUM_PAGE)});
$strType, 'add file', {oExpectedManifest => \%oManifest, strOptionalParam => '--checksum-page'});
# Selective Restore
#---------------------------------------------------------------------------------------------------------------------------
@ -906,7 +900,7 @@ sub run
{&MANIFEST_SUBKEY_CHECKSUM});
$oHostDbMaster->restore(
'selective restore 16384', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'selective restore 16384', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true,
strOptionalParam => '--db-include=16384'});
@ -924,7 +918,7 @@ sub run
delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/16384/17000'}{&MANIFEST_SUBKEY_CHECKSUM});
$oHostDbMaster->restore(
'selective restore 32768', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'selective restore 32768', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true,
strOptionalParam => '--db-include=32768'});
@ -932,12 +926,12 @@ sub run
'7579ada0808d7f98087a0a586d0df9de009cdc33';
$oHostDbMaster->restore(
'error on invalid id', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'error on invalid id', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true,
iExpectedExitStatus => ERROR_DB_MISSING, strOptionalParam => '--log-level-console=warn --db-include=7777'});
$oHostDbMaster->restore(
'error on system id', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'error on system id', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true,
iExpectedExitStatus => ERROR_DB_INVALID, strOptionalParam => '--log-level-console=warn --db-include=1'});
@ -952,7 +946,7 @@ sub run
delete($oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'});
$oHostDbMaster->restore(
'no tablespace remap', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'no tablespace remap', 'latest',
{rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bTablespace => false,
strOptionalParam => '--tablespace-map-all=../../tablespace'});
@ -965,8 +959,9 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
if (!$bRemote && !$bS3)
{
executeTest('ls -1Rtr ' . storageRepo()->pathGet('backup/' . $self->stanza() . '/' . PATH_BACKUP_HISTORY),
{oLogTest => $self->expect(), bRemote => $bRemote});
executeTest(
'ls -1Rtr ' . $oHostBackup->repoBackupPath(PATH_BACKUP_HISTORY),
{oLogTest => $self->expect(), bRemote => $bRemote});
}
# Test backup from standby warning that standby not configured so option reset
@ -975,7 +970,7 @@ sub run
{
$strBackup = $oHostBackup->backup(
$strType, 'option backup-standby reset - backup performed from master', {oExpectedManifest => \%oManifest,
strOptionalParam => '--log-level-console=info --' . cfgOptionName(CFGOPT_BACKUP_STANDBY)});
strOptionalParam => '--log-level-console=info --backup-standby'});
}
}
}

15
test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm
View File

@ -20,12 +20,11 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::VmTest;
@ -86,7 +85,7 @@ sub run
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
# Create the archive info file
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-online'});
# Push a WAL segment
&log(INFO, ' push first WAL');
@ -96,7 +95,7 @@ sub run
if ($iError == 0)
{
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
$oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'},
&INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}});
}
@ -120,13 +119,12 @@ sub run
# Fix the database version
if ($iError == 0)
{
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE));
}
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageRepo()->list(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))},
"000000010000000100000001-${strWalHash}${strCompressExt}",
'segment 2-4 not pushed', {iWaitSeconds => 5});
@ -134,8 +132,7 @@ sub run
$oHostDbMaster->archivePush($strWalPath, $strWalTestFile, 5);
$self->testResult(
sub {storageRepo()->list(
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))},
"(000000010000000100000001-${strWalHash}${strCompressExt}, " .
"000000010000000100000005-${strWalHash}${strCompressExt})",
'segment 5 is pushed', {iWaitSeconds => 5});

41
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
View File

@ -20,11 +20,10 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
@ -38,13 +37,15 @@ use pgBackRestTest::Common::VmTest;
sub archiveCheck
{
my $self = shift;
my $oHostBackup = shift;
my $strArchiveFile = shift;
my $strArchiveChecksum = shift;
my $strCompressType = shift;
my $strSpoolPath = shift;
# Build the archive name to check for at the destination
my $strArchiveCheck = PG_VERSION_94 . "-1/${strArchiveFile}-${strArchiveChecksum}";
my $strArchiveCheck = $oHostBackup->repoArchivePath(
PG_VERSION_94 . "-1/" . substr($strArchiveFile, 0, 16) . "/${strArchiveFile}-${strArchiveChecksum}");
if (defined($strCompressType))
{
@ -56,13 +57,13 @@ sub archiveCheck
do
{
$bFound = storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}");
$bFound = storageRepo()->exists($strArchiveCheck);
}
while (!$bFound && waitMore($oWait));
if (!$bFound)
{
confess 'unable to find ' . storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}");
confess "unable to find ${strArchiveCheck}";
}
if (defined($strSpoolPath))
@ -108,7 +109,7 @@ sub run
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE});
# Reduce console logging to detail
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(DETAIL)}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(DETAIL)}});
# Create the wal path
my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
@ -121,11 +122,11 @@ sub run
# Create archive-push command
my $strCommandPush =
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=' . $self->stanza() .
' ' . cfgCommandName(CFGCMD_ARCHIVE_PUSH);
' archive-push';
my $strCommandGet =
$oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=' . $self->stanza() .
' ' . cfgCommandName(CFGCMD_ARCHIVE_GET);
' archive-get';
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' archive.info missing');
@ -142,9 +143,7 @@ sub run
{iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()});
#---------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaCreate(
'stanza create',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('stanza create', {strOptionalParam => '--no-online'});
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' push first WAL');
@ -160,7 +159,7 @@ sub run
push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}");
# Test that the WAL was pushed
$self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType);
$self->archiveCheck($oHostBackup, $strSourceFile, $strArchiveChecksum, $strCompressType);
# Remove from archive_status
storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready");
@ -245,7 +244,7 @@ sub run
}
# Test that the WAL was pushed
$self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbMaster->spoolPath());
$self->archiveCheck($oHostBackup, $strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbMaster->spoolPath());
# Remove from archive_status
storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready");
@ -263,7 +262,7 @@ sub run
"${strCommandPush} --archive-async ${strWalPath}/00000002.history",
{oLogTest => $self->expect()});
if (!storageRepo()->exists(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/00000002.history'))
if (!storageRepo()->exists($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/00000002.history')))
{
confess 'unable to find history file in archive';
}
@ -275,7 +274,7 @@ sub run
# db section and corresponding history munged
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
$oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB_HISTORY => {'1' => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}});
$oHostDbMaster->executeSimple(
@ -283,13 +282,13 @@ sub run
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
# Restore the file to its original condition
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE));
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' db system-id mismatch error');
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
$oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000},
&INFO_ARCHIVE_SECTION_DB_HISTORY => {'1' => {&INFO_ARCHIVE_KEY_DB_ID => 5000900090001855000}}});
@ -302,7 +301,7 @@ sub run
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
# Restore the file to its original condition
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE));
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' stop');
@ -384,7 +383,7 @@ sub run
$oHostDbMaster->executeSimple(
$strCommandPush . " ${strWalPath}/${strSourceFile}.partial",
{oLogTest => $self->expect()});
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum);
$self->archiveCheck($oHostBackup, "${strSourceFile}.partial", $strArchiveChecksum);
push(@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}");
@ -393,7 +392,7 @@ sub run
$oHostDbMaster->executeSimple(
$strCommandPush . " ${strWalPath}/${strSourceFile}.partial", {oLogTest => $self->expect()});
$self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum);
$self->archiveCheck($oHostBackup, "${strSourceFile}.partial", $strArchiveChecksum);
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' .partial WAL with different checksum');
@ -405,7 +404,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')},
sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))},
'(' . join(', ', @stryExpectedWAL) . ')',
'all WAL in archive', {iWaitSeconds => 5});
}

47
test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm
View File

@ -21,50 +21,17 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::VmTest;
use pgBackRestTest::Env::ExpireEnvTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostS3Test;
use pgBackRestTest::Env::HostEnvTest;
####################################################################################################################################
# initStanzaOption
####################################################################################################################################
sub initStanzaOption
{
my $self = shift;
my $strDbBasePath = shift;
my $strRepoPath = shift;
my $oHostS3 = shift;
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_PG_PATH, $strDbBasePath);
$self->optionTestSet(CFGOPT_REPO_PATH, $strRepoPath);
$self->optionTestSet(CFGOPT_LOG_PATH, $self->testPath());
$self->optionTestSetBool(CFGOPT_ONLINE, false);
$self->optionTestSet(CFGOPT_DB_TIMEOUT, 5);
$self->optionTestSet(CFGOPT_PROTOCOL_TIMEOUT, 6);
if (defined($oHostS3))
{
$self->optionTestSet(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3);
$self->optionTestSet(CFGOPT_REPO_S3_KEY, HOST_S3_ACCESS_KEY);
$self->optionTestSet(CFGOPT_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY);
$self->optionTestSet(CFGOPT_REPO_S3_BUCKET, HOST_S3_BUCKET);
$self->optionTestSet(CFGOPT_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT);
$self->optionTestSet(CFGOPT_REPO_S3_REGION, HOST_S3_REGION);
$self->optionTestSet(CFGOPT_REPO_S3_HOST, $oHostS3->ipGet());
$self->optionTestSetBool(CFGOPT_REPO_S3_VERIFY_TLS, false);
}
}
####################################################################################################################################
# run
####################################################################################################################################
@ -103,12 +70,9 @@ sub run
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
true, $self->expect(), {bS3 => $bS3, bRepoEncrypt => $bEncrypt});
$self->initStanzaOption($oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3);
$self->configTestLoad(CFGCMD_STANZA_CREATE);
# Create the test object
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest(
$oHostBackup, $self->backrestExe(), storageRepo(), $self->expect(), $self);
$oHostBackup, $self->backrestExe(), storageRepo(), $oHostDbMaster->dbPath(), $self->expect(), $self);
$oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_92);
@ -202,12 +166,9 @@ sub run
my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
true, $self->expect(), {bS3 => $bS3, bRepoEncrypt => $bEncrypt});
$self->initStanzaOption($oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3);
$self->configTestLoad(CFGCMD_STANZA_CREATE);
# Create the test object
my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest(
$oHostBackup, $self->backrestExe(), storageRepo(), $self->expect(), $self);
$oHostBackup, $self->backrestExe(), storageRepo(), $oHostDbMaster->dbPath(), $self->expect(), $self);
$oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_92);

62
test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm
View File

@ -19,14 +19,13 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
@ -40,17 +39,6 @@ sub run
{
my $self = shift;
# Archive and backup info file names
my $strArchiveInfoFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE;
my $strArchiveInfoCopyFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT;
my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old";
my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old";
my $strBackupInfoFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO;
my $strBackupInfoCopyFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT;
my $strBackupInfoOldFile = "${strBackupInfoFile}.old";
my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old";
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => true, compress => GZ},
@ -80,9 +68,20 @@ sub run
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt,
strCompressType => $strCompressType});
# Archive and backup info file names
my $strArchiveInfoFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE);
my $strArchiveInfoCopyFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE . INI_COPY_EXT);
my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old";
my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old";
my $strBackupInfoFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO);
my $strBackupInfoCopyFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO . INI_COPY_EXT);
my $strBackupInfoOldFile = "${strBackupInfoFile}.old";
my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old";
# Create the stanza
$oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_MISSING,
strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_LOG_LEVEL_FILE) . '=info'});
strOptionalParam => '--no-online --log-level-file=info'});
# Generate pg_control for stanza-create
storageTest()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
@ -91,17 +90,17 @@ sub run
# Fail stanza upgrade before stanza-create has been performed
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaUpgrade('fail on stanza not initialized since archive.info is missing',
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-online'});
# Create the stanza successfully without force
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-online'});
# Rerun stanza-create and confirm it does not fail
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaCreate(
'do not fail on rerun of stanza-create - info files exist and DB section ok',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
{strOptionalParam => '--no-online'});
# Stanza Create fails when not using force - database mismatch with pg_control file
#--------------------------------------------------------------------------------------------------------------------------
@ -109,15 +108,14 @@ sub run
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
$oHostBackup->stanzaCreate('fail on database mismatch and warn force option deprecated',
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) .
' --' . cfgOptionName(CFGOPT_FORCE)});
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-online --force'});
# Restore pg_control
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_93);
# Perform a stanza upgrade which will indicate already up to date
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaUpgrade('already up to date', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaUpgrade('already up to date', {strOptionalParam => '--no-online'});
# Create the wal path
my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
@ -140,7 +138,7 @@ sub run
if (!$bEncrypt)
{
$oHostBackup->stanzaCreate('fail on archive info file missing from non-empty dir',
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-online'});
}
# Restore info files from copy
@ -170,8 +168,7 @@ sub run
# Save a pre-upgrade copy of archive info fo testing db-id mismatch
forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false});
$oHostBackup->stanzaUpgrade('successful upgrade creates additional history', {strOptionalParam => '--no-' .
cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaUpgrade('successful upgrade creates additional history', {strOptionalParam => '--no-online'});
# Make sure that WAL from the old version can still be retrieved
#--------------------------------------------------------------------------------------------------------------------------
@ -193,7 +190,7 @@ sub run
# Push a WAL segment so have a valid file in the latest DB archive dir only
$oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1);
$self->testResult(
sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-2/0000000100000001')},
sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-2/0000000100000001'))},
'000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_94) . ".${strCompressType}",
'check that WAL is in the archive at -2');
@ -201,8 +198,7 @@ sub run
#--------------------------------------------------------------------------------------------------------------------------
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_PGTBLSPC);
$oHostBackup->backup(
'full', 'create first full backup ',
{strOptionalParam => '--repo1-retention-full=2 --no-' . cfgOptionName(CFGOPT_ONLINE)}, false);
'full', 'create first full backup ', {strOptionalParam => '--repo1-retention-full=2 --no-online'}, false);
# Upgrade the stanza
#--------------------------------------------------------------------------------------------------------------------------
@ -210,8 +206,7 @@ sub run
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_95);
forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
$oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-online'});
# Copy archive.info and restore really old version
forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false});
@ -219,16 +214,15 @@ sub run
forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoFile, {bRecurse => false});
# Confirm versions
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza()));
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
my $oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath());
my $oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath());
$self->testResult(sub {$oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
PG_VERSION_93)}, true, 'archive at old pg version');
$self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef,
PG_VERSION_95)}, true, 'backup at new pg version');
$oHostBackup->stanzaUpgrade(
'upgrade fails with mismatched db-ids',
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
'upgrade fails with mismatched db-ids', {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-online'});
# Restore archive.info
forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false});
@ -239,8 +233,8 @@ sub run
$oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1);
# Test backup is changed from type=DIFF to FULL (WARN message displayed)
my $oExecuteBackup = $oHostBackup->backupBegin('diff', 'diff changed to full backup',
{strOptionalParam => '--repo1-retention-full=2 --no-' . cfgOptionName(CFGOPT_ONLINE)});
my $oExecuteBackup = $oHostBackup->backupBegin(
'diff', 'diff changed to full backup', {strOptionalParam => '--repo1-retention-full=2 --no-online'});
$oHostBackup->backupEnd('full', $oExecuteBackup, undef, false);
# Delete the stanza

8
test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm
View File

@ -16,7 +16,6 @@ use Storable qw(dclone);
use Time::HiRes qw(gettimeofday);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
@ -63,11 +62,8 @@ sub run
for (my $iIndex = 0; $iIndex < $iRunTotal; $iIndex++)
{
executeTest(
$self->backrestExe() . ' --' . cfgOptionName(CFGOPT_STANZA) . '=' . $self->stanza() .
' --' . cfgOptionName(CFGOPT_ARCHIVE_ASYNC) .
' --' . cfgOptionName(CFGOPT_SPOOL_PATH) . '=' . $self->{strSpoolPath} .
' --' . cfgOptionName(CFGOPT_ARCHIVE_TIMEOUT) . '=1' .
' archive-push /pg_xlog/000000010000000100000001');
$self->backrestExe() . ' --stanza=' . $self->stanza() . ' --archive-async --spool-path=' . $self->{strSpoolPath} .
' --archive-timeout=1 archive-push /pg_xlog/000000010000000100000001');
}
&log(INFO, 'time per execution: ' . ((gettimeofday() - $lTimeBegin) / $iRunTotal));

98
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
View File

@ -20,10 +20,8 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
@ -111,8 +109,8 @@ sub run
# without slowing down the other tests too much.
if ($bS3)
{
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
$oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}});
}
$oHostDbMaster->clusterCreate();
@ -122,7 +120,7 @@ sub run
# Get passphrase to access the Manifest file from backup.info - returns undefined if repo not encrypted
my $strCipherPass =
(new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP)))->cipherPassSub();
(new pgBackRest::Backup::Info($oHostBackup->repoBackupPath()))->cipherPassSub();
# Create a manifest with the pg version to get version-specific paths
my $oManifest = new pgBackRest::Manifest(BOGUS, {bLoad => false, strDbVersion => $self->pgVersion(),
@ -157,13 +155,13 @@ sub run
my $strComment = undef;
# Archive and backup info file names
my $strArchiveInfoFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE;
my $strArchiveInfoCopyFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT;
my $strArchiveInfoFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE);
my $strArchiveInfoCopyFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE . INI_COPY_EXT);
my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old";
my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old";
my $strBackupInfoFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO;
my $strBackupInfoCopyFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT;
my $strBackupInfoFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO);
my $strBackupInfoCopyFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO . INI_COPY_EXT);
my $strBackupInfoOldFile = "${strBackupInfoFile}.old";
my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old";
@ -232,7 +230,7 @@ sub run
# load the archive info file and munge it for testing by breaking the database version
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
$oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'},
&INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}});
@ -245,7 +243,7 @@ sub run
}
# Restore the file to its original condition
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
$oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE));
# Check archive_timeout error when WAL segment is not found
$strComment = 'fail on archive timeout';
@ -268,7 +266,7 @@ sub run
# Load the backup.info file and munge it for testing by breaking the database version and system id
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
$oHostBackup->repoBackupPath(FILE_BACKUP_INFO),
{&INFO_BACKUP_SECTION_DB =>
{&INFO_BACKUP_KEY_DB_VERSION => '8.0', &INFO_BACKUP_KEY_SYSTEM_ID => 6999999999999999999},
&INFO_BACKUP_SECTION_DB_HISTORY =>
@ -284,7 +282,7 @@ sub run
}
# Restore the file to its original condition
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO));
$oHostBackup->infoRestore($oHostBackup->repoBackupPath(FILE_BACKUP_INFO));
# ??? Removed temporarily until manifest build can be brought back into the check command
# Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check
@ -332,22 +330,20 @@ sub run
# Run stanza-create online to confirm proper handling of configValidation error against new pg-path
$oHostBackup->stanzaCreate('fail on database mismatch with directory',
{strOptionalParam => ' --' . cfgOptionName(CFGOPT_PG_PATH) . '=' . $oHostDbMaster->dbPath() .
'/testbase/', iExpectedExitStatus => ERROR_DB_MISMATCH});
{strOptionalParam => ' --pg1-path=' . $oHostDbMaster->dbPath() . '/testbase/',
iExpectedExitStatus => ERROR_DB_MISMATCH});
# Remove the directories to be able to create the stanza
forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP, {bRecurse => true});
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE, {bRecurse => true});
forceStorageRemove(storageRepo(), $oHostBackup->repoBackupPath(), {bRecurse => true});
forceStorageRemove(storageRepo(), $oHostBackup->repoArchivePath(), {bRecurse => true});
# Stanza Upgrade - tests configValidate code - all other tests in synthetic integration tests
#-----------------------------------------------------------------------------------------------------------------------
# Run stanza-create offline to create files needing to be upgraded (using new pg-path)
$oHostBackup->stanzaCreate('successfully create stanza files to be upgraded',
{strOptionalParam =>
' --' . cfgOptionName(CFGOPT_PG_PATH) . '=' . $oHostDbMaster->dbPath() .
'/testbase/ --no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza()));
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
{strOptionalParam => ' --pg1-path=' . $oHostDbMaster->dbPath() . '/testbase/ --no-online --force'});
my $oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath());
my $oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath());
# Read info files to confirm the files were created with a different database version
if ($self->pgVersion() eq PG_VERSION_94)
@ -369,8 +365,8 @@ sub run
$oHostBackup->stanzaUpgrade('upgrade stanza files online');
# Reread the info files and confirm the result
$oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza()));
$oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
$oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath());
$oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath());
$self->testResult(sub {$oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
$self->pgVersion())}, true, 'archive upgrade online corrects db');
$self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef,
@ -409,10 +405,10 @@ sub run
my $strFullBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'update during backup',
{strOptionalParam => ' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=16384'});
{strOptionalParam => ' --buffer-size=16384'});
# Enabled async archiving
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_ARCHIVE_ASYNC) => 'y'}});
$oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'archive-async' => 'y'}});
# Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it
# down enough to make it evident that the async process is working.
@ -444,7 +440,7 @@ sub run
}
$oHostDbStandby->restore(
'restore backup on replica', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
'restore backup on replica', 'latest',
{rhRemapHash => \%oRemapHash, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY,
strOptionalParam =>
' --recovery-option="primary_conninfo=host=' . HOST_DB_MASTER .
@ -470,19 +466,13 @@ sub run
{
my $strStandbyBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to reach master',
{bStandby => true,
iExpectedExitStatus => ERROR_DB_CONNECT,
strOptionalParam => '--' .
cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, cfgOptionIndexTotal(CFGOPT_PG_PATH))) . '=' . BOGUS});
{bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg8-host=' . BOGUS});
}
else
{
my $strStandbyBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to access at least one standby',
{bStandby => true,
iExpectedExitStatus => ERROR_DB_CONNECT,
strOptionalParam => '--' .
cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, cfgOptionIndexTotal(CFGOPT_PG_PATH))) . '=' . BOGUS});
{bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg8-host=' . BOGUS});
}
}
@ -490,7 +480,7 @@ sub run
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby',
{bStandby => true,
iExpectedExitStatus => $oHostDbStandby->pgVersion() >= PG_VERSION_BACKUP_STANDBY ? undef : ERROR_CONFIG,
strOptionalParam => '--' . cfgOptionName(CFGOPT_REPO_RETENTION_FULL) . '=1'});
strOptionalParam => '--repo1-retention-full=1'});
if ($oHostDbStandby->pgVersion() >= PG_VERSION_BACKUP_STANDBY)
{
@ -632,9 +622,7 @@ sub run
# Exercise --delta checksum option
my $strIncrBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'update during backup',
{strOptionalParam =>
'--' . cfgOptionName(CFGOPT_STOP_AUTO) . ' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=32768 --delta'});
CFGOPTVAL_BACKUP_TYPE_INCR, 'update during backup', {strOptionalParam => '--stop-auto --buffer-size=32768 --delta'});
# Ensure the check command runs properly with a tablespace unless there is a bogus host
if (!$oHostBackup->bogusHost())
@ -702,9 +690,7 @@ sub run
if ($bTestLocal)
{
# Expect failure because postmaster.pid exists
$oHostDbMaster->restore(
'postmaster running', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
{iExpectedExitStatus => ERROR_POSTMASTER_RUNNING});
$oHostDbMaster->restore('postmaster running', 'latest', {iExpectedExitStatus => ERROR_POSTMASTER_RUNNING});
}
$oHostDbMaster->clusterStop();
@ -712,8 +698,7 @@ sub run
if ($bTestLocal)
{
# Expect failure because db path is not empty
$oHostDbMaster->restore(
'path not empty', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
$oHostDbMaster->restore('path not empty', 'latest', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
}
# Drop and recreate db path
@ -726,7 +711,7 @@ sub run
# Now the restore should work
$oHostDbMaster->restore(
undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
undef, 'latest',
{strOptionalParam => ($bTestLocal ? ' --db-include=test2 --db-include=test3' : '') . ' --buffer-size=16384'});
# Test that the first database has not been restored since --db-include did not include test1
@ -784,8 +769,8 @@ sub run
{
# Backup info will have the catalog number
my $oBackupInfo = new pgBackRest::Common::Ini(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO),
{bLoad => false, strContent => ${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)}});
storageRepo(), $oHostBackup->repoBackupPath(FILE_BACKUP_INFO),
{bLoad => false, strContent => ${storageRepo()->get($oHostBackup->repoBackupPath(FILE_BACKUP_INFO))}});
# Construct the special path
$strTablespacePath .=
@ -899,8 +884,7 @@ sub run
storageTest()->put($oHostDbMaster->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL);
}
$oHostDbMaster->restore(
undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE});
$oHostDbMaster->restore(undef, 'latest', {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE});
$oHostDbMaster->clusterStart();
$oHostDbMaster->sqlSelectOneTest('select message from test', $strXidMessage);
@ -916,7 +900,7 @@ sub run
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
undef, 'latest',
{bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_TIME, strTarget => $strTimeTarget,
strTargetAction => $oHostDbMaster->pgVersion() >= PG_VERSION_91 ? 'promote' : undef,
strTargetTimeline => $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'current' : undef,
@ -952,7 +936,7 @@ sub run
$oHostDbMaster->clusterStop();
$oHostDbMaster->restore(
undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET),
undef, 'latest',
{bDelta => true, bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_NAME, strTarget => $strNameTarget,
strTargetAction => 'promote',
strTargetTimeline => $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'current' : undef});
@ -997,15 +981,13 @@ sub run
# Incr backup - make sure a --no-online backup fails
#-----------------------------------------------------------------------------------------------------------------------
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on --no-' . cfgOptionName(CFGOPT_ONLINE),
{iExpectedExitStatus => ERROR_POSTMASTER_RUNNING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on --no-online',
{iExpectedExitStatus => ERROR_POSTMASTER_RUNNING, strOptionalParam => '--no-online'});
# Incr backup - allow --no-online backup to succeed with --force
#-----------------------------------------------------------------------------------------------------------------------
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR,
'succeed on --no-' . cfgOptionName(CFGOPT_ONLINE) . ' with --' . cfgOptionName(CFGOPT_FORCE),
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
CFGOPTVAL_BACKUP_TYPE_INCR, 'succeed on --no-online with --force', {strOptionalParam => '--no-online --force'});
}
# Stanza-delete --force without access to pgbackrest on database host
@ -1017,8 +999,8 @@ sub run
{
$oHostDbMaster->stop();
$oHostBackup->stop({strStanza => $self->stanza});
$oHostBackup->stanzaDelete("delete stanza with --force when pgbackrest on pg host not accessible",
{strOptionalParam => ' --' . cfgOptionName(CFGOPT_FORCE)});
$oHostBackup->stanzaDelete(
"delete stanza with --force when pgbackrest on pg host not accessible", {strOptionalParam => ' --force'});
$oHostDbMaster->start();
$oHostBackup->start();
}

181
test/test.pl
View File

@ -81,10 +81,9 @@ test.pl [options]
--c-only only run C tests
--container-only only run tests that must be run in a container
--gen-only only run auto-generation
--gen-libc generate libc code to embed in the binary
--no-gen do not run code generation
--code-count generate code counts
--smart perform libc/package builds only when source timestamps have changed
--smart perform bin/package builds only when source timestamps have changed
--no-package do not build packages
--dev --smart --no-package --no-optimize
--dev-test --no-package
@ -155,7 +154,6 @@ my $bNoCoverage = false;
my $bCOnly = false;
my $bContainerOnly = false;
my $bGenOnly = false;
my $bGenLibC = false;
my $bNoGen = false;
my $bCodeCount = false;
my $bSmart = false;
@ -206,7 +204,6 @@ GetOptions ('q|quiet' => \$bQuiet,
'c-only' => \$bCOnly,
'container-only' => \$bContainerOnly,
'gen-only' => \$bGenOnly,
'gen-libc' => \$bGenLibC,
'no-gen' => \$bNoGen,
'code-count' => \$bCodeCount,
'smart' => \$bSmart,
@ -369,6 +366,7 @@ eval
# Get the base backrest path
my $strBackRestBase = dirname(dirname(abs_path($0)));
my $strVagrantPath = "${strBackRestBase}/test/.vagrant";
my $oStorageBackRest = new pgBackRestTest::Common::Storage(
$strBackRestBase, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false}));
@ -550,28 +548,9 @@ eval
}
}
# Auto-generate Perl code
#-----------------------------------------------------------------------------------------------------------------------
use lib dirname(dirname($0)) . '/libc/build/lib';
use pgBackRestLibC::Build;
if (!$bSmart || grep(/^(build|libc\/build)\//, @stryModifiedList))
{
errorDefineLoad(${$oStorageBackRest->get("build/error.yaml")});
my @stryBuilt = buildXsAll("${strBackRestBase}/libc");
&log(INFO, " autogenerated Perl code: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes'));
if (@stryBuilt)
{
push(@stryBuiltAll, @stryBuilt);
push(@stryModifiedList, @stryBuilt);
}
}
# Auto-generate C Makefile
#-----------------------------------------------------------------------------------------------------------------------
if (!$bSmart || grep(/^(src|libc)\//, @stryModifiedList))
if (!$bSmart || grep(/^src\//, @stryModifiedList))
{
my @stryBuilt;
my $strBuilt = 'src/Makefile.in';
@ -713,8 +692,7 @@ eval
#---------------------------------------------------------------------------------------------------------------------------
my $oyTestRun;
my $bBinRequired = $bBuildOnly;
my $bLibCHostRequired = $bBuildOnly;
my $bLibCVmRequired = $bBuildOnly;
my $bHostBinRequired = $bBuildOnly;
# Only get the test list when they can run
if (!$bBuildOnly)
@ -732,37 +710,26 @@ eval
$bBinRequired = true;
}
# Host LibC required if a Perl test
# Host bin required if a Perl test
if (!$hTest->{&TEST_C})
{
$bLibCHostRequired = true;
}
# VM LibC required if Perl and not an integration test
if (!$hTest->{&TEST_C} && !$hTest->{&TEST_INTEGRATION})
{
$bLibCVmRequired = true;
$bHostBinRequired = true;
}
}
}
my $strBuildRequired;
if ($bBinRequired || $bLibCHostRequired || $bLibCVmRequired)
if ($bBinRequired || $bHostBinRequired)
{
if ($bBinRequired)
{
$strBuildRequired = "bin";
}
if ($bLibCHostRequired)
if ($bHostBinRequired)
{
$strBuildRequired .= ", libc host";
}
if ($bLibCVmRequired)
{
$strBuildRequired .= ", libc vm";
$strBuildRequired .= ", bin host";
}
}
else
@ -777,9 +744,8 @@ eval
if (!$bDryRun)
{
my $oVm = vmGet();
my $strVagrantPath = "${strBackRestBase}/test/.vagrant";
my $lTimestampLast;
my @stryBinSrcPath = ('src', 'libc');
my @stryBinSrcPath = ('src');
my $strBinPath = "${strVagrantPath}/bin";
my $rhBinBuild = {};
@ -794,6 +760,12 @@ eval
my $bLogDetail = $strLogLevel eq 'detail';
my @stryBuildVm = $strVm eq VM_ALL ? VM_LIST : ($strVm);
# Build binary for the host
if ($bHostBinRequired)
{
push(@stryBuildVm, VM_NONE);
}
foreach my $strBuildVM (@stryBuildVm)
{
my $strBuildPath = "${strBinPath}/${strBuildVM}/src";
@ -802,7 +774,7 @@ eval
# Build configure/compile options and see if they have changed from the previous build
my $strCFlags =
"-Wfatal-errors -g -fPIC -D_FILE_OFFSET_BITS=64" .
"-Wfatal-errors -g" .
(vmWithBackTrace($strBuildVM) && $bBackTrace ? ' -DWITH_BACKTRACE' : '') .
($bDebugTestTrace ? ' -DDEBUG_TEST_TRACE' : '');
my $strLdFlags = vmWithBackTrace($strBuildVM) && $bBackTrace ? '-lbacktrace' : '';
@ -872,114 +844,6 @@ eval
}
}
# Build the C Library
#-----------------------------------------------------------------------------------------------------------------------
if ($bLibCHostRequired || $bLibCVmRequired)
{
my $strLibCPath = "${strVagrantPath}/bin";
# Loop through VMs to do the C Library builds
my $bLogDetail = $strLogLevel eq 'detail';
my @stryBuildVm = ();
if ($strVm eq VM_ALL)
{
@stryBuildVm = $bLibCVmRequired ? VM_LIST : ($strVmHost);
}
else
{
@stryBuildVm = $bLibCVmRequired && $strVmHost ne $strVm ? ($strVmHost, $strVm) : ($strVmHost);
}
foreach my $strBuildVM (@stryBuildVm)
{
my $strBuildPath = "${strLibCPath}/${strBuildVM}/libc";
my $bContainerExists = $strBuildVM ne $strVmHost && $strBuildVM ne VM_NONE;
my $strConfigOptions = (vmDebugIntegration($strBuildVM) ? ' --enable-test' : '');
my $strLibCSmart = "${strBuildPath}/blib/arch/auto/pgBackRest/LibC/LibC.so";
my $bRebuild = !$bSmart;
# Rebuild if the modification time of the smart file does equal the last changes in source paths
if ($bSmart)
{
if (!$oStorageBackRest->exists($strLibCSmart) ||
$oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast)
{
&log(INFO, " libc dependencies have changed for ${strBuildVM}, rebuilding...");
$bRebuild = true;
}
}
if ($bRebuild)
{
&log(INFO, " build test library for ${strBuildVM} (${strBuildPath})");
if (!$rhBinBuild->{$strBuildVM})
{
foreach my $strBinSrcPath (@stryBinSrcPath)
{
$oStorageBackRest->pathCreate(
"${strBinPath}/${strBuildVM}/${strBinSrcPath}", {bIgnoreExists => true, bCreateParent => true});
}
executeTest(
"rsync -rt" . (!$bSmart ? " --delete-excluded" : '') .
" --include=" . join('/*** --include=', @stryBinSrcPath) . '/*** --exclude=*' .
" ${strBackRestBase}/ ${strBinPath}/${strBuildVM}");
}
# Can't reuse any object files in the libc dir because it does not have proper dependencies
executeTest(
"rsync -rt --exclude=Makefile --delete ${strBackRestBase}/libc/ ${strLibCPath}/${strBuildVM}/libc");
# It's very expensive to rebuild the Makefile so make sure it has actually changed
my $bMakeRebuild =
!$oStorageBackRest->exists("${strBuildPath}/Makefile") ||
($oStorageBackRest->info("${strBackRestBase}/libc/Makefile.PL")->mtime >
$oStorageBackRest->info("${strBuildPath}/Makefile.PL")->mtime);
if ($bContainerExists)
{
executeTest(
"docker run -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-build",
{bSuppressStdErr => true});
}
if ($bMakeRebuild)
{
&log(INFO, " rebuild test library Makefile for ${strBuildVM}");
executeTest(
($bContainerExists ? "docker exec -i test-build bash -c '" : '') .
"cd ${strBuildPath} && perl ${strBuildPath}/Makefile.PL INSTALLMAN1DIR=none INSTALLMAN3DIR=none" .
($bContainerExists ? "'" : ''),
{bSuppressStdErr => true, bShowOutputAsync => $bLogDetail});
}
if (!$oStorageBackRest->exists("${strLibCPath}/${strBuildVM}/src/Makefile"))
{
executeTest(
($bContainerExists ? 'docker exec -i test-build ' : '') .
"bash -c 'cd ${strLibCPath}/${strBuildVM}/src && ./configure${strConfigOptions}'",
{bShowOutputAsync => $bLogDetail});
}
executeTest(
($bContainerExists ? 'docker exec -i test-build ' : '') .
"make -j ${iBuildMax} --silent --directory ${strBuildPath}",
{bShowOutputAsync => $bLogDetail});
if ($bContainerExists)
{
executeTest("docker rm -f test-build");
}
}
}
}
# Build the package
#-----------------------------------------------------------------------------------------------------------------------
if (!$bNoPackage && $strVm ne VM_NONE)
@ -1125,7 +989,7 @@ eval
executeTest(
"tar --transform='s_^_pgbackrest-release-${strVersionBase}/_'" .
" -czf ${strBuildPath}/SOURCES/${strVersionBase}.tar.gz -C ${strBackRestBase}" .
" lib libc src LICENSE");
" src LICENSE");
# Copy package files
executeTest(
@ -1463,13 +1327,6 @@ eval
################################################################################################################################
# Runs tests
################################################################################################################################
push(
@INC,
"${strBackRestBase}/test/.vagrant/bin/" .
((testDefModuleTest($stryModule[0], $stryModuleTest[0]))->{&TESTDEF_CONTAINER} ? $strVm : $strVmHost) .
"/libc/blib/arch");
buildLoadLibC();
my $iRun = 0;
# Create host group for containers
@ -1487,7 +1344,7 @@ eval
$strBackRestBase, # Base backrest directory
$strTestPath, # Path where the tests will run
'/usr/bin/' . PROJECT_EXE, # Path to the backrest executable
"${strBackRestBase}/bin/" . PROJECT_EXE, # Path to the backrest Perl helper
"${strVagrantPath}/bin/" . VM_NONE . '/src/' . PROJECT_EXE, # Path to the backrest Perl storage helper
$strPgVersion ne 'minimal' ? $strPgSqlBin: undef, # Pg bin path
$strPgVersion ne 'minimal' ? $strPgVersion: undef, # Pg version
$stryModule[0], $stryModuleTest[0], \@iyModuleTestRun, # Module info