diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index 596f9b8c1..e39a0a9ee 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -730,9 +730,6 @@ mkdir -p /build/pgbackrest-release-{[version]} - - cp -r {[pgbackrest-repo-path]}/libc /build/pgbackrest-release-{[version]} - cp -r {[pgbackrest-repo-path]}/src /build/pgbackrest-release-{[version]} diff --git a/lib/pgBackRest/Archive/Common.pm b/lib/pgBackRest/Archive/Common.pm index 1b2a86325..b66b9c6ab 100644 --- a/lib/pgBackRest/Archive/Common.pm +++ b/lib/pgBackRest/Archive/Common.pm @@ -17,8 +17,6 @@ use pgBackRest::DbVersion; use pgBackRest::Common::Exception; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; #################################################################################################################################### diff --git a/lib/pgBackRest/Archive/Info.pm b/lib/pgBackRest/Archive/Info.pm index 381540db0..c73e4a6e6 100644 --- a/lib/pgBackRest/Archive/Info.pm +++ b/lib/pgBackRest/Archive/Info.pm @@ -19,13 +19,11 @@ use File::Basename qw(dirname basename); use pgBackRest::Archive::Common; use pgBackRest::Common::Exception; -use pgBackRest::Config::Config; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::DbVersion; use pgBackRest::InfoCommon; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Base; use pgBackRest::Storage::Helper; @@ -95,9 +93,9 @@ sub new # Init object and store variables eval { - $self = $class->SUPER::new($strArchiveInfoFile, {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, - oStorage => storageRepo(), strCipherPass => storageRepo()->cipherPassUser(), - strCipherPassSub => $strCipherPassSub}); + $self = $class->SUPER::new( + storageRepo(), $strArchiveInfoFile, + {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub}); return true; } or do diff --git a/lib/pgBackRest/Backup/Common.pm b/lib/pgBackRest/Backup/Common.pm index 82f20bb34..6dc6dfe28 100644 --- a/lib/pgBackRest/Backup/Common.pm +++ b/lib/pgBackRest/Backup/Common.pm @@ -14,10 +14,8 @@ use File::Basename; use pgBackRest::Common::Log; use pgBackRest::Common::String; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; -use pgBackRest::Protocol::Storage::Helper; -use pgBackRest::Storage::Helper; use pgBackRest::Manifest; +use pgBackRest::Storage::Helper; #################################################################################################################################### # Latest backup link constant @@ -25,6 +23,13 @@ use pgBackRest::Manifest; use constant LINK_LATEST => 'latest'; push @EXPORT, qw(LINK_LATEST); +use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; + push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); +use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; + push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); +use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; + push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); + #################################################################################################################################### # backupRegExpGet # @@ -200,6 +205,7 @@ sub backupLabel ( $strOperation, $oStorageRepo, + $strRepoBackupPath, $strType, $strBackupLabelLast, $lTimestampStart @@ -208,6 +214,7 @@ sub backupLabel ( __PACKAGE__ . '::backupLabelFormat', \@_, {name => 'oStorageRepo', trace => true}, + {name => 'strRepoBackupPath', trace => true}, {name => 'strType', trace => true}, {name => 'strBackupLabelLast', required => false, trace => true}, {name => 'lTimestampStart', trace => true} @@ -221,15 +228,15 @@ sub backupLabel # clocks. In practice this is most useful for making offline testing faster since it allows the wait after manifest build to # be skipped by dealing with any backup label collisions here. if ($oStorageRepo->list( - STORAGE_REPO_BACKUP, + $strRepoBackupPath, {strExpression => ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStart) . ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? 'F' : '(D|I)$')}) || $oStorageRepo->list( - STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTimestampStart), + "${strRepoBackupPath}/" . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTimestampStart), {strExpression => ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStart) . - ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? 'F' : '(D|I)\.manifest\.' . COMPRESS_EXT . qw{$}), + ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? 'F' : '(D|I)\.manifest\.gz$'), bIgnoreMissing => true})) { waitRemainder(); diff --git a/lib/pgBackRest/Backup/Info.pm b/lib/pgBackRest/Backup/Info.pm index 14578b17a..80bf83ae3 100644 --- a/lib/pgBackRest/Backup/Info.pm +++ b/lib/pgBackRest/Backup/Info.pm @@ -19,10 +19,8 @@ use pgBackRest::Backup::Common; use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; -use pgBackRest::Config::Config; use pgBackRest::InfoCommon; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; #################################################################################################################################### @@ -137,9 +135,9 @@ sub new # Init object and store variables eval { - $self = $class->SUPER::new($strBackupInfoFile, {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, - oStorage => $oStorage, strCipherPass => $oStorage->cipherPassUser(), - strCipherPassSub => $strCipherPassSub}); + $self = $class->SUPER::new( + $oStorage, $strBackupInfoFile, + {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub}); return true; } or do diff --git a/lib/pgBackRest/Common/Exception.pm b/lib/pgBackRest/Common/Exception.pm index c277ba0db..f4e16093f 100644 --- a/lib/pgBackRest/Common/Exception.pm +++ b/lib/pgBackRest/Common/Exception.pm @@ -12,12 +12,98 @@ use Scalar::Util qw(blessed); use Exporter qw(import); our @EXPORT = qw(); -use pgBackRest::Common::ExceptionAuto; +#################################################################################################################################### +# Error Definitions +#################################################################################################################################### +use constant ERROR_MINIMUM => 25; +push @EXPORT, qw(ERROR_MINIMUM); +use constant ERROR_MAXIMUM => 125; +push @EXPORT, qw(ERROR_MAXIMUM); -#################################################################################################################################### -# Export error constants -#################################################################################################################################### -push(@EXPORT, @pgBackRest::Common::ExceptionAuto::EXPORT); +use constant ERROR_ASSERT => 25; +push @EXPORT, qw(ERROR_ASSERT); +use constant ERROR_CHECKSUM => 26; +push @EXPORT, qw(ERROR_CHECKSUM); +use constant ERROR_CONFIG => 27; +push @EXPORT, qw(ERROR_CONFIG); +use constant ERROR_FILE_INVALID => 28; +push @EXPORT, qw(ERROR_FILE_INVALID); +use constant ERROR_FORMAT => 29; +push @EXPORT, qw(ERROR_FORMAT); +use constant ERROR_OPTION_INVALID_VALUE => 32; +push @EXPORT, qw(ERROR_OPTION_INVALID_VALUE); +use constant ERROR_POSTMASTER_RUNNING => 38; +push @EXPORT, qw(ERROR_POSTMASTER_RUNNING); +use constant ERROR_PATH_NOT_EMPTY => 40; +push @EXPORT, qw(ERROR_PATH_NOT_EMPTY); +use constant ERROR_FILE_OPEN => 41; +push @EXPORT, qw(ERROR_FILE_OPEN); +use constant ERROR_FILE_READ => 42; +push @EXPORT, qw(ERROR_FILE_READ); +use constant ERROR_ARCHIVE_MISMATCH => 44; +push @EXPORT, qw(ERROR_ARCHIVE_MISMATCH); +use constant ERROR_ARCHIVE_DUPLICATE => 45; +push @EXPORT, qw(ERROR_ARCHIVE_DUPLICATE); +use constant ERROR_PATH_CREATE => 47; +push @EXPORT, qw(ERROR_PATH_CREATE); +use constant ERROR_LOCK_ACQUIRE => 50; +push @EXPORT, qw(ERROR_LOCK_ACQUIRE); +use constant ERROR_BACKUP_MISMATCH => 51; +push @EXPORT, qw(ERROR_BACKUP_MISMATCH); +use constant ERROR_PATH_OPEN => 53; +push @EXPORT, qw(ERROR_PATH_OPEN); +use constant ERROR_PATH_SYNC => 54; +push @EXPORT, qw(ERROR_PATH_SYNC); +use constant ERROR_FILE_MISSING => 55; +push @EXPORT, qw(ERROR_FILE_MISSING); +use constant ERROR_DB_CONNECT => 56; +push @EXPORT, qw(ERROR_DB_CONNECT); +use constant ERROR_DB_QUERY => 57; +push @EXPORT, qw(ERROR_DB_QUERY); +use constant ERROR_DB_MISMATCH => 58; +push @EXPORT, qw(ERROR_DB_MISMATCH); +use constant ERROR_PATH_REMOVE => 61; +push @EXPORT, qw(ERROR_PATH_REMOVE); +use constant ERROR_STOP => 62; +push @EXPORT, qw(ERROR_STOP); +use constant ERROR_FILE_WRITE => 64; +push @EXPORT, qw(ERROR_FILE_WRITE); +use constant ERROR_FEATURE_NOT_SUPPORTED => 67; +push @EXPORT, qw(ERROR_FEATURE_NOT_SUPPORTED); +use constant ERROR_ARCHIVE_COMMAND_INVALID => 68; +push @EXPORT, qw(ERROR_ARCHIVE_COMMAND_INVALID); +use constant ERROR_LINK_EXPECTED => 69; +push @EXPORT, qw(ERROR_LINK_EXPECTED); +use constant ERROR_LINK_DESTINATION => 70; +push @EXPORT, qw(ERROR_LINK_DESTINATION); +use constant ERROR_PATH_MISSING => 73; +push @EXPORT, qw(ERROR_PATH_MISSING); +use constant ERROR_FILE_MOVE => 74; +push @EXPORT, qw(ERROR_FILE_MOVE); +use constant ERROR_PATH_TYPE => 77; +push @EXPORT, qw(ERROR_PATH_TYPE); +use constant ERROR_DB_MISSING => 80; +push @EXPORT, qw(ERROR_DB_MISSING); +use constant ERROR_DB_INVALID => 81; +push @EXPORT, qw(ERROR_DB_INVALID); +use constant ERROR_ARCHIVE_TIMEOUT => 82; +push @EXPORT, qw(ERROR_ARCHIVE_TIMEOUT); +use constant ERROR_ARCHIVE_DISABLED => 87; +push @EXPORT, qw(ERROR_ARCHIVE_DISABLED); +use constant ERROR_FILE_OWNER => 88; +push @EXPORT, qw(ERROR_FILE_OWNER); +use constant ERROR_PATH_EXISTS => 92; +push @EXPORT, qw(ERROR_PATH_EXISTS); +use constant ERROR_FILE_EXISTS => 93; +push @EXPORT, qw(ERROR_FILE_EXISTS); +use constant ERROR_CRYPTO => 95; +push @EXPORT, qw(ERROR_CRYPTO); +use constant ERROR_INVALID => 123; +push @EXPORT, qw(ERROR_INVALID); +use constant ERROR_UNHANDLED => 124; +push @EXPORT, qw(ERROR_UNHANDLED); +use constant ERROR_UNKNOWN => 125; +push @EXPORT, qw(ERROR_UNKNOWN); #################################################################################################################################### # CONSTRUCTOR diff --git a/lib/pgBackRest/Common/ExceptionAuto.pm b/lib/pgBackRest/Common/ExceptionAuto.pm deleted file mode 100644 index 73e30b981..000000000 --- a/lib/pgBackRest/Common/ExceptionAuto.pm +++ /dev/null @@ -1,183 +0,0 @@ -#################################################################################################################################### -# COMMON EXCEPTION AUTO MODULE -# -# Automatically generated by Build.pm -- do not modify directly. -#################################################################################################################################### -package pgBackRest::Common::ExceptionAuto; - -use strict; -use warnings FATAL => qw(all); - -use Exporter qw(import); - our @EXPORT = qw(); - -#################################################################################################################################### -# Error Definitions -#################################################################################################################################### -use constant ERROR_MINIMUM => 25; -push @EXPORT, qw(ERROR_MINIMUM); -use constant ERROR_MAXIMUM => 125; -push @EXPORT, qw(ERROR_MAXIMUM); - -use constant ERROR_ASSERT => 25; -push @EXPORT, qw(ERROR_ASSERT); -use constant ERROR_CHECKSUM => 26; -push @EXPORT, qw(ERROR_CHECKSUM); -use constant ERROR_CONFIG => 27; -push @EXPORT, qw(ERROR_CONFIG); -use constant ERROR_FILE_INVALID => 28; -push @EXPORT, qw(ERROR_FILE_INVALID); -use constant ERROR_FORMAT => 29; -push @EXPORT, qw(ERROR_FORMAT); -use constant ERROR_COMMAND_REQUIRED => 30; -push @EXPORT, qw(ERROR_COMMAND_REQUIRED); -use constant ERROR_OPTION_INVALID => 31; -push @EXPORT, qw(ERROR_OPTION_INVALID); -use constant ERROR_OPTION_INVALID_VALUE => 32; -push @EXPORT, qw(ERROR_OPTION_INVALID_VALUE); -use constant ERROR_OPTION_INVALID_RANGE => 33; -push @EXPORT, qw(ERROR_OPTION_INVALID_RANGE); -use constant ERROR_OPTION_INVALID_PAIR => 34; -push @EXPORT, qw(ERROR_OPTION_INVALID_PAIR); -use constant ERROR_OPTION_DUPLICATE_KEY => 35; -push @EXPORT, qw(ERROR_OPTION_DUPLICATE_KEY); -use constant ERROR_OPTION_NEGATE => 36; -push @EXPORT, qw(ERROR_OPTION_NEGATE); -use constant ERROR_OPTION_REQUIRED => 37; -push @EXPORT, qw(ERROR_OPTION_REQUIRED); -use constant ERROR_POSTMASTER_RUNNING => 38; -push @EXPORT, qw(ERROR_POSTMASTER_RUNNING); -use constant ERROR_PROTOCOL => 39; -push @EXPORT, qw(ERROR_PROTOCOL); -use constant ERROR_PATH_NOT_EMPTY => 40; -push @EXPORT, qw(ERROR_PATH_NOT_EMPTY); -use constant ERROR_FILE_OPEN => 41; -push @EXPORT, qw(ERROR_FILE_OPEN); -use constant ERROR_FILE_READ => 42; -push @EXPORT, qw(ERROR_FILE_READ); -use constant ERROR_PARAM_REQUIRED => 43; -push @EXPORT, qw(ERROR_PARAM_REQUIRED); -use constant ERROR_ARCHIVE_MISMATCH => 44; -push @EXPORT, qw(ERROR_ARCHIVE_MISMATCH); -use constant ERROR_ARCHIVE_DUPLICATE => 45; -push @EXPORT, qw(ERROR_ARCHIVE_DUPLICATE); -use constant ERROR_VERSION_NOT_SUPPORTED => 46; -push @EXPORT, qw(ERROR_VERSION_NOT_SUPPORTED); -use constant ERROR_PATH_CREATE => 47; -push @EXPORT, qw(ERROR_PATH_CREATE); -use constant ERROR_COMMAND_INVALID => 48; -push @EXPORT, qw(ERROR_COMMAND_INVALID); -use constant ERROR_HOST_CONNECT => 49; -push @EXPORT, qw(ERROR_HOST_CONNECT); -use constant ERROR_LOCK_ACQUIRE => 50; -push @EXPORT, qw(ERROR_LOCK_ACQUIRE); -use constant ERROR_BACKUP_MISMATCH => 51; -push @EXPORT, qw(ERROR_BACKUP_MISMATCH); -use constant ERROR_FILE_SYNC => 52; -push @EXPORT, qw(ERROR_FILE_SYNC); -use constant ERROR_PATH_OPEN => 53; -push @EXPORT, qw(ERROR_PATH_OPEN); -use constant ERROR_PATH_SYNC => 54; -push @EXPORT, qw(ERROR_PATH_SYNC); -use constant ERROR_FILE_MISSING => 55; -push @EXPORT, qw(ERROR_FILE_MISSING); -use constant ERROR_DB_CONNECT => 56; -push @EXPORT, qw(ERROR_DB_CONNECT); -use constant ERROR_DB_QUERY => 57; -push @EXPORT, qw(ERROR_DB_QUERY); -use constant ERROR_DB_MISMATCH => 58; -push @EXPORT, qw(ERROR_DB_MISMATCH); -use constant ERROR_DB_TIMEOUT => 59; -push @EXPORT, qw(ERROR_DB_TIMEOUT); -use constant ERROR_FILE_REMOVE => 60; -push @EXPORT, qw(ERROR_FILE_REMOVE); -use constant ERROR_PATH_REMOVE => 61; -push @EXPORT, qw(ERROR_PATH_REMOVE); -use constant ERROR_STOP => 62; -push @EXPORT, qw(ERROR_STOP); -use constant ERROR_TERM => 63; -push @EXPORT, qw(ERROR_TERM); -use constant ERROR_FILE_WRITE => 64; -push @EXPORT, qw(ERROR_FILE_WRITE); -use constant ERROR_PROTOCOL_TIMEOUT => 66; -push @EXPORT, qw(ERROR_PROTOCOL_TIMEOUT); -use constant ERROR_FEATURE_NOT_SUPPORTED => 67; -push @EXPORT, qw(ERROR_FEATURE_NOT_SUPPORTED); -use constant ERROR_ARCHIVE_COMMAND_INVALID => 68; -push @EXPORT, qw(ERROR_ARCHIVE_COMMAND_INVALID); -use constant ERROR_LINK_EXPECTED => 69; -push @EXPORT, qw(ERROR_LINK_EXPECTED); -use constant ERROR_LINK_DESTINATION => 70; -push @EXPORT, qw(ERROR_LINK_DESTINATION); -use constant ERROR_HOST_INVALID => 72; -push @EXPORT, qw(ERROR_HOST_INVALID); -use constant ERROR_PATH_MISSING => 73; -push @EXPORT, qw(ERROR_PATH_MISSING); -use constant ERROR_FILE_MOVE => 74; -push @EXPORT, qw(ERROR_FILE_MOVE); -use constant ERROR_BACKUP_SET_INVALID => 75; -push @EXPORT, qw(ERROR_BACKUP_SET_INVALID); -use constant ERROR_TABLESPACE_MAP => 76; -push @EXPORT, qw(ERROR_TABLESPACE_MAP); -use constant ERROR_PATH_TYPE => 77; -push @EXPORT, qw(ERROR_PATH_TYPE); -use constant ERROR_LINK_MAP => 78; -push @EXPORT, qw(ERROR_LINK_MAP); -use constant ERROR_FILE_CLOSE => 79; -push @EXPORT, qw(ERROR_FILE_CLOSE); -use constant ERROR_DB_MISSING => 80; -push @EXPORT, qw(ERROR_DB_MISSING); -use constant ERROR_DB_INVALID => 81; -push @EXPORT, qw(ERROR_DB_INVALID); -use constant ERROR_ARCHIVE_TIMEOUT => 82; -push @EXPORT, qw(ERROR_ARCHIVE_TIMEOUT); -use constant ERROR_FILE_MODE => 83; -push @EXPORT, qw(ERROR_FILE_MODE); -use constant ERROR_OPTION_MULTIPLE_VALUE => 84; -push @EXPORT, qw(ERROR_OPTION_MULTIPLE_VALUE); -use constant ERROR_PROTOCOL_OUTPUT_REQUIRED => 85; -push @EXPORT, qw(ERROR_PROTOCOL_OUTPUT_REQUIRED); -use constant ERROR_LINK_OPEN => 86; -push @EXPORT, qw(ERROR_LINK_OPEN); -use constant ERROR_ARCHIVE_DISABLED => 87; -push @EXPORT, qw(ERROR_ARCHIVE_DISABLED); -use constant ERROR_FILE_OWNER => 88; -push @EXPORT, qw(ERROR_FILE_OWNER); -use constant ERROR_USER_MISSING => 89; -push @EXPORT, qw(ERROR_USER_MISSING); -use constant ERROR_OPTION_COMMAND => 90; -push @EXPORT, qw(ERROR_OPTION_COMMAND); -use constant ERROR_GROUP_MISSING => 91; -push @EXPORT, qw(ERROR_GROUP_MISSING); -use constant ERROR_PATH_EXISTS => 92; -push @EXPORT, qw(ERROR_PATH_EXISTS); -use constant ERROR_FILE_EXISTS => 93; -push @EXPORT, qw(ERROR_FILE_EXISTS); -use constant ERROR_MEMORY => 94; -push @EXPORT, qw(ERROR_MEMORY); -use constant ERROR_CRYPTO => 95; -push @EXPORT, qw(ERROR_CRYPTO); -use constant ERROR_PARAM_INVALID => 96; -push @EXPORT, qw(ERROR_PARAM_INVALID); -use constant ERROR_PATH_CLOSE => 97; -push @EXPORT, qw(ERROR_PATH_CLOSE); -use constant ERROR_FILE_INFO => 98; -push @EXPORT, qw(ERROR_FILE_INFO); -use constant ERROR_JSON_FORMAT => 99; -push @EXPORT, qw(ERROR_JSON_FORMAT); -use constant ERROR_KERNEL => 100; -push @EXPORT, qw(ERROR_KERNEL); -use constant ERROR_SERVICE => 101; -push @EXPORT, qw(ERROR_SERVICE); -use constant ERROR_EXECUTE => 102; -push @EXPORT, qw(ERROR_EXECUTE); -use constant ERROR_RUNTIME => 122; -push @EXPORT, qw(ERROR_RUNTIME); -use constant ERROR_INVALID => 123; -push @EXPORT, qw(ERROR_INVALID); -use constant ERROR_UNHANDLED => 124; -push @EXPORT, qw(ERROR_UNHANDLED); -use constant ERROR_UNKNOWN => 125; -push @EXPORT, qw(ERROR_UNKNOWN); - -1; diff --git a/lib/pgBackRest/Common/Ini.pm b/lib/pgBackRest/Common/Ini.pm index cd1502c20..2562ee064 100644 --- a/lib/pgBackRest/Common/Ini.pm +++ b/lib/pgBackRest/Common/Ini.pm @@ -74,17 +74,13 @@ sub new my $self = {}; bless $self, $class; - # Load Storage::Helper module - require pgBackRest::Storage::Helper; - pgBackRest::Storage::Helper->import(); - # Assign function parameters, defaults, and log debug info ( my $strOperation, + $self->{oStorage}, $self->{strFileName}, my $bLoad, my $strContent, - $self->{oStorage}, $self->{iInitFormat}, $self->{strInitVersion}, my $bIgnoreMissing, @@ -94,10 +90,10 @@ sub new logDebugParam ( __PACKAGE__ . '->new', \@_, + {name => 'oStorage', trace => true}, {name => 'strFileName', trace => true}, {name => 'bLoad', optional => true, default => true, trace => true}, {name => 'strContent', optional => true, trace => true}, - {name => 'oStorage', optional => true, default => storageLocal(), trace => true}, {name => 'iInitFormat', optional => true, default => REPOSITORY_FORMAT, trace => true}, {name => 'strInitVersion', optional => true, default => PROJECT_VERSION, trace => true}, {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, @@ -105,11 +101,6 @@ sub new {name => 'strCipherPassSub', optional => true, trace => true}, ); - if (defined($self->{oStorage}->cipherPassUser()) && !defined($self->{strCipherPass})) - { - confess &log(ERROR, 'passphrase is required when storage is encrypted', ERROR_CRYPTO); - } - # Set changed to false $self->{bModified} = false; @@ -139,11 +130,6 @@ sub new { $self->set(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS, undef, $strCipherPassSub); } - elsif ((defined($self->{strCipherPass}) && !defined($strCipherPassSub)) || - (!defined($self->{strCipherPass}) && defined($strCipherPassSub))) - { - confess &log(ASSERT, 'a user passphrase and sub passphrase are both required when encrypting'); - } } return $self; diff --git a/lib/pgBackRest/Common/Io/Process.pm b/lib/pgBackRest/Common/Io/Process.pm new file mode 100644 index 000000000..528fdb6cb --- /dev/null +++ b/lib/pgBackRest/Common/Io/Process.pm @@ -0,0 +1,192 @@ +#################################################################################################################################### +# Process Execution, Management, and IO +#################################################################################################################################### +package pgBackRest::Common::Io::Process; +use parent 'pgBackRest::Common::Io::Filter'; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Exporter qw(import); + our @EXPORT = qw(); +use IPC::Open3 qw(open3); +use POSIX qw(:sys_wait_h); +use Symbol 'gensym'; + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Io::Buffered; +use pgBackRest::Common::Log; +use pgBackRest::Common::Wait; + +#################################################################################################################################### +# Amount of time to attempt to retrieve errors when a process terminates unexpectedly +#################################################################################################################################### +use constant IO_ERROR_TIMEOUT => 5; + +#################################################################################################################################### +# new - use open3 to run the command and get the io handles +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $oParent, + $strCommand, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'oParent', trace => true}, + {name => 'strCommand', trace => true}, + ); + + # Bless with new class + my $self = $class->SUPER::new($oParent); + bless $self, $class; + + # Use open3 to run the command + my ($iProcessId, $fhRead, $fhWrite, $fhReadError); + $fhReadError = gensym; + + $iProcessId = IPC::Open3::open3($fhWrite, $fhRead, $fhReadError, $strCommand); + + # Set handles + $self->handleReadSet($fhRead); + $self->handleWriteSet($fhWrite); + + # Set variables + $self->{iProcessId} = $iProcessId; + $self->{fhReadError} = $fhReadError; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self} + ); +} + +#################################################################################################################################### +# error - handle errors +#################################################################################################################################### +sub error +{ + my $self = shift; + my $iCode = shift; + my $strMessage = shift; + my $strDetail = shift; + my $bClose = shift; + + if (defined($self->{iProcessId})) + { + my $oWait = waitInit(defined($iCode) ? IO_ERROR_TIMEOUT : 0); + + do + { + # Check the result + my $iResult = waitpid($self->{iProcessId}, $bClose ? 0 : WNOHANG); + + # Error if the process exited unexpectedly + if ($iResult != 0) + { + # Get the exit status + $self->{iExitStatus} = $iResult == -1 ? 255 : ${^CHILD_ERROR_NATIVE} >> 8; + + # Drain the stderr stream + my $strError; + my $oIoError = new pgBackRest::Common::Io::Buffered( + new pgBackRest::Common::Io::Handle($self->id(), $self->{fhReadError}), 5, $self->bufferMax()); + + while (defined(my $strLine = $oIoError->readLine(true, false))) + { + $strError .= (defined($strError) ? "\n" : '') . $strLine; + } + + delete($self->{iProcessId}); + + if ((!$bClose && $self->{iExitStatus} != 0) || defined($strError)) + { + my $iErrorCode = + $self->{iExitStatus} >= ERROR_MINIMUM && $self->{iExitStatus} <= ERROR_MAXIMUM ? + $self->{iExitStatus} : ERROR_FILE_READ; + + logErrorResult( + $iErrorCode, $self->id() . ' terminated unexpectedly' . + ($self->{iExitStatus} != 255 ? sprintf(' [%03d]', $self->{iExitStatus}) : ''), + $strError); + } + } + } + while (waitMore($oWait)); + + if (defined($iCode)) + { + $self->parent()->error($iCode, $strMessage, $strDetail); + } + } + else + { + confess &log(ASSERT, 'cannot call error() after process has been closed'); + } +} + +#################################################################################################################################### +# Get process id +#################################################################################################################################### +sub processId +{ + my $self = shift; + + return $self->{iProcessId}; +} + +#################################################################################################################################### +# Get exit status (after close() is called) +#################################################################################################################################### +sub exitStatus +{ + my $self = shift; + + return $self->{iExitStatus}; +} + +#################################################################################################################################### +# writeLine - check for error before writing line +#################################################################################################################################### +sub writeLine +{ + my $self = shift; + my $strBuffer = shift; + + # Check if the process has exited abnormally (doesn't seem like we should need this, but the next syswrite does a hard + # abort if the remote process has already closed) + $self->error(); + + return $self->parent()->writeLine($strBuffer); +} + +#################################################################################################################################### +# close - check if the process terminated on error +#################################################################################################################################### +sub close +{ + my $self = shift; + + if (defined($self->{iProcessId})) + { + $self->error(undef, undef, undef, true); + + # Class parent close + $self->parent()->close(); + } + + return true; +} + +1; diff --git a/lib/pgBackRest/Config/Config.pm b/lib/pgBackRest/Config/Config.pm deleted file mode 100644 index ab96d75d5..000000000 --- a/lib/pgBackRest/Config/Config.pm +++ /dev/null @@ -1,453 +0,0 @@ -#################################################################################################################################### -# CONFIG MODULE -#################################################################################################################################### -package pgBackRest::Config::Config; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use JSON::PP; - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Ini; -use pgBackRest::Common::Io::Base; -use pgBackRest::Common::Log; -use pgBackRest::Common::String; -use pgBackRest::Common::Wait; -use pgBackRest::LibC qw(:config :configDefine); -use pgBackRest::Version; - -#################################################################################################################################### -# Export config constants and functions -#################################################################################################################################### -push(@EXPORT, @{$pgBackRest::LibC::EXPORT_TAGS{config}}); -push(@EXPORT, @{$pgBackRest::LibC::EXPORT_TAGS{configDefine}}); - -#################################################################################################################################### -# SOURCE Constants -#################################################################################################################################### -use constant CFGDEF_SOURCE_CONFIG => 'config'; - push @EXPORT, qw(CFGDEF_SOURCE_CONFIG); -use constant CFGDEF_SOURCE_PARAM => 'param'; - push @EXPORT, qw(CFGDEF_SOURCE_PARAM); -use constant CFGDEF_SOURCE_DEFAULT => 'default'; - push @EXPORT, qw(CFGDEF_SOURCE_DEFAULT); - -#################################################################################################################################### -# Configuration section constants -#################################################################################################################################### -use constant CFGDEF_SECTION_GLOBAL => 'global'; - push @EXPORT, qw(CFGDEF_SECTION_GLOBAL); -use constant CFGDEF_SECTION_STANZA => 'stanza'; - push @EXPORT, qw(CFGDEF_SECTION_STANZA); - -#################################################################################################################################### -# Module variables -#################################################################################################################################### -my %oOption; # Option hash -my $strCommand; # Command (backup, archive-get, ...) -my $bInitLog = false; # Has logging been initialized yet? - -#################################################################################################################################### -# configLogging - configure logging based on options -#################################################################################################################################### -sub configLogging -{ - my $bLogInitForce = shift; - - if ($bInitLog || (defined($bLogInitForce) && $bLogInitForce)) - { - logLevelSet( - cfgOptionValid(CFGOPT_LOG_LEVEL_FILE) ? cfgOption(CFGOPT_LOG_LEVEL_FILE) : OFF, - cfgOptionValid(CFGOPT_LOG_LEVEL_CONSOLE) ? cfgOption(CFGOPT_LOG_LEVEL_CONSOLE) : OFF, - cfgOptionValid(CFGOPT_LOG_LEVEL_STDERR) ? cfgOption(CFGOPT_LOG_LEVEL_STDERR) : OFF, - cfgOptionValid(CFGOPT_LOG_TIMESTAMP) ? cfgOption(CFGOPT_LOG_TIMESTAMP) : undef, - cfgOptionValid(CFGOPT_PROCESS_MAX) ? cfgOption(CFGOPT_PROCESS_MAX) : undef); - - $bInitLog = true; - } -} - -push @EXPORT, qw(configLogging); - -#################################################################################################################################### -# Load configuration -# -# Additional conditions that cannot be codified by the option definitions are also tested here. -#################################################################################################################################### -sub configLoad -{ - my $bInitLogging = shift; - my $strBackRestBin = shift; - my $strCommandName = shift; - my $rstrConfigJson = shift; - - # Set backrest bin - projectBinSet($strBackRestBin); - - # Set command - $strCommand = $strCommandName; - - eval - { - %oOption = %{(JSON::PP->new()->allow_nonref())->decode($$rstrConfigJson)}; - return true; - } - or do - { - confess &log(ASSERT, "unable to parse config JSON"); - }; - - # Load options into final option hash - for (my $iOptionId = 0; $iOptionId < cfgOptionTotal(); $iOptionId++) - { - my $strOptionName = cfgOptionName($iOptionId); - - # If option is defined it is valid - if (defined($oOption{$strOptionName})) - { - # Convert JSON bool to standard bool that Perl understands - if (cfgDefOptionType($iOptionId) eq CFGDEF_TYPE_BOOLEAN && defined($oOption{$strOptionName}{value})) - { - $oOption{$strOptionName}{value} = $oOption{$strOptionName}{value} eq INI_TRUE ? true : false; - } - } - # Else it is not valid - else - { - $oOption{$strOptionName}{valid} = false; - } - } - - # If this is not the remote and logging is allowed (to not overwrite log levels for tests) then set the log level so that - # INFO/WARN messages can be displayed (the user may still disable them). This should be run before any WARN logging is - # generated. - if (!defined($bInitLogging) || $bInitLogging) - { - configLogging(true); - } - - return true; -} - -push @EXPORT, qw(configLoad); - -#################################################################################################################################### -# cfgOptionIdFromIndex - return name for options that can be indexed (e.g. pg1-host, pg2-host). -#################################################################################################################################### -sub cfgOptionIdFromIndex -{ - my $iOptionId = shift; - my $iIndex = shift; - - # If the option doesn't have a prefix it can't be indexed - $iIndex = defined($iIndex) ? $iIndex : 1; - my $strPrefix = cfgDefOptionPrefix($iOptionId); - - if (!defined($strPrefix)) - { - if ($iIndex > 1) - { - confess &log(ASSERT, "'" . cfgOptionName($iOptionId) . "' option does not allow indexing"); - } - - return $iOptionId; - } - - return cfgOptionId("${strPrefix}${iIndex}" . substr(cfgOptionName($iOptionId), index(cfgOptionName($iOptionId), '-'))); -} - -push @EXPORT, qw(cfgOptionIdFromIndex); - -#################################################################################################################################### -# cfgOptionSource - how was the option set? -#################################################################################################################################### -sub cfgOptionSource -{ - my $iOptionId = shift; - - cfgOptionValid($iOptionId, true); - - return $oOption{cfgOptionName($iOptionId)}{source}; -} - -push @EXPORT, qw(cfgOptionSource); - -#################################################################################################################################### -# cfgOptionValid - is the option valid for the current command? -#################################################################################################################################### -sub cfgOptionValid -{ - my $iOptionId = shift; - my $bError = shift; - - # If defined then this is the command help is being generated for so all valid checks should be against that command - my $iCommandId; - - if (defined($strCommand)) - { - $iCommandId = cfgCommandId($strCommand); - } - - if (defined($iCommandId) && cfgDefOptionValid($iCommandId, $iOptionId)) - { - return true; - } - - if (defined($bError) && $bError) - { - my $strOption = cfgOptionName($iOptionId); - - if (!defined($oOption{$strOption})) - { - confess &log(ASSERT, "option '${strOption}' does not exist"); - } - - confess &log(ASSERT, "option '${strOption}' not valid for command '" . cfgCommandName(cfgCommandGet()) . "'"); - } - - return false; -} - -push @EXPORT, qw(cfgOptionValid); - -#################################################################################################################################### -# cfgOption - get option value -#################################################################################################################################### -sub cfgOption -{ - my $iOptionId = shift; - my $bRequired = shift; - - cfgOptionValid($iOptionId, true); - - my $strOption = cfgOptionName($iOptionId); - - if (!defined($oOption{$strOption}{value}) && (!defined($bRequired) || $bRequired)) - { - confess &log(ASSERT, "option ${strOption} is required"); - } - - return $oOption{$strOption}{value}; -} - -push @EXPORT, qw(cfgOption); - -#################################################################################################################################### -# cfgOptionDefault - get option default value -#################################################################################################################################### -sub cfgOptionDefault -{ - my $iOptionId = shift; - - cfgOptionValid($iOptionId, true); - - return cfgDefOptionDefault(cfgCommandId($strCommand), $iOptionId); -} - -push @EXPORT, qw(cfgOptionDefault); - -#################################################################################################################################### -# cfgOptionSet - set option value and source -#################################################################################################################################### -sub cfgOptionSet -{ - my $iOptionId = shift; - my $oValue = shift; - my $bForce = shift; - - my $strOption = cfgOptionName($iOptionId); - - if (!cfgOptionValid($iOptionId, !defined($bForce) || !$bForce)) - { - $oOption{$strOption}{valid} = true; - } - - $oOption{$strOption}{source} = CFGDEF_SOURCE_PARAM; - $oOption{$strOption}{value} = $oValue; -} - -push @EXPORT, qw(cfgOptionSet); - -#################################################################################################################################### -# cfgOptionTest - test if an option exists or has a specific value -#################################################################################################################################### -sub cfgOptionTest -{ - my $iOptionId = shift; - my $strValue = shift; - - if (!cfgOptionValid($iOptionId)) - { - return false; - } - - if (defined($strValue)) - { - return cfgOption($iOptionId) eq $strValue ? true : false; - } - - return defined($oOption{cfgOptionName($iOptionId)}{value}) ? true : false; -} - -push @EXPORT, qw(cfgOptionTest); - -#################################################################################################################################### -# cfgCommandGet - get the current command -#################################################################################################################################### -sub cfgCommandGet -{ - return cfgCommandId($strCommand); -} - -push @EXPORT, qw(cfgCommandGet); - -#################################################################################################################################### -# cfgCommandTest - test that the current command is equal to the provided value -#################################################################################################################################### -sub cfgCommandTest -{ - my $iCommandIdTest = shift; - - return cfgCommandName($iCommandIdTest) eq $strCommand; -} - -push @EXPORT, qw(cfgCommandTest); - -#################################################################################################################################### -# Set current command -#################################################################################################################################### -sub cfgCommandSet -{ - my $iCommandId = shift; - - $strCommand = cfgCommandName($iCommandId); -} - -push @EXPORT, qw(cfgCommandSet); - -#################################################################################################################################### -# cfgCommandWrite - using the options for the current command, write the command string for another command -# -# For example, this can be used to write the archive-get command for recovery.conf during a restore. -#################################################################################################################################### -sub cfgCommandWrite -{ - my $iNewCommandId = shift; - my $bIncludeConfig = shift; - my $strExeString = shift; - my $bIncludeCommand = shift; - my $oOptionOverride = shift; - my $bDisplayOnly = shift; - - # Set defaults - $strExeString = defined($strExeString) ? $strExeString : projectBin(); - $bIncludeConfig = defined($bIncludeConfig) ? $bIncludeConfig : false; - $bIncludeCommand = defined($bIncludeCommand) ? $bIncludeCommand : true; - - # Iterate the options to figure out which ones are not default and need to be written out to the new command string - for (my $iOptionId = 0; $iOptionId < cfgOptionTotal(); $iOptionId++) - { - my $strOption = cfgOptionName($iOptionId); - my $bSecure = cfgDefOptionSecure($iOptionId); - - # Skip option if it is secure and should not be output in logs or the command line - next if (($bSecure || $iOptionId == CFGOPT_REPO_CIPHER_TYPE) && !$bDisplayOnly); - - # Process any option id overrides first - if (defined($oOptionOverride->{$iOptionId})) - { - if (defined($oOptionOverride->{$iOptionId}{value})) - { - $strExeString .= cfgCommandWriteOptionFormat( - $strOption, false, $bSecure, {value => $oOptionOverride->{$iOptionId}{value}}); - } - } - # And process overrides passed by string - this is used by Perl compatibility functions - elsif (defined($oOptionOverride->{$strOption})) - { - if (defined($oOptionOverride->{$strOption}{value})) - { - $strExeString .= cfgCommandWriteOptionFormat( - $strOption, false, $bSecure, {value => $oOptionOverride->{$strOption}{value}}); - } - } - # else look for non-default options in the current configuration - elsif (cfgDefOptionValid($iNewCommandId, $iOptionId) && - defined($oOption{$strOption}{value}) && - ($bIncludeConfig ? - $oOption{$strOption}{source} ne CFGDEF_SOURCE_DEFAULT : $oOption{$strOption}{source} eq CFGDEF_SOURCE_PARAM)) - { - my $oValue; - my $bMulti = false; - - # If this is a hash then it will break up into multiple command-line options - if (ref($oOption{$strOption}{value}) eq 'HASH') - { - $oValue = $oOption{$strOption}{value}; - $bMulti = true; - } - # Else a single value but store it in a hash anyway to make processing below simpler - else - { - $oValue = {value => $oOption{$strOption}{value}}; - } - - $strExeString .= cfgCommandWriteOptionFormat($strOption, $bMulti, $bSecure, $oValue); - } - # Else is reset - elsif (cfgDefOptionValid($iNewCommandId, $iOptionId) && $oOption{$strOption}{reset}) - { - $strExeString .= " --reset-${strOption}"; - } - } - - if ($bIncludeCommand) - { - $strExeString .= ' ' . cfgCommandName($iNewCommandId); - } - - return $strExeString; -} - -push @EXPORT, qw(cfgCommandWrite); - -# Helper function for cfgCommandWrite() to correctly format options for command-line usage -sub cfgCommandWriteOptionFormat -{ - my $strOption = shift; - my $bMulti = shift; - my $bSecure = shift; - my $oValue = shift; - - # Loops though all keys in the hash - my $strOptionFormat = ''; - my $strParam; - - foreach my $strKey (sort(keys(%$oValue))) - { - # Get the value - if the original value was a hash then the key must be prefixed - my $strValue = $bSecure ? '' : ($bMulti ? "${strKey}=" : '') . $$oValue{$strKey}; - - # Handle the no- prefix for boolean values - if (cfgDefOptionType(cfgOptionId($strOption)) eq CFGDEF_TYPE_BOOLEAN) - { - $strParam = '--' . ($strValue ? '' : 'no-') . $strOption; - } - else - { - $strParam = "--${strOption}=${strValue}"; - } - - # Add quotes if the value has spaces in it - $strOptionFormat .= ' ' . (index($strValue, " ") != -1 ? "\"${strParam}\"" : $strParam); - } - - return $strOptionFormat; -} - -1; diff --git a/lib/pgBackRest/LibC.pm b/lib/pgBackRest/LibC.pm deleted file mode 100644 index 6ccb0d1ca..000000000 --- a/lib/pgBackRest/LibC.pm +++ /dev/null @@ -1,64 +0,0 @@ -#################################################################################################################################### -# C to Perl Interface -#################################################################################################################################### -package pgBackRest::LibC; -use base 'Exporter'; - -use 5.010001; -use strict; -use warnings; -use Carp; - -use pgBackRest::LibCAuto; - -# Dynamically create constants -my $rhConstant = pgBackRest::LibCAuto::libcAutoConstant(); - -foreach my $strConstant (keys(%{$rhConstant})) -{ - eval "use constant ${strConstant} => '" . $rhConstant->{$strConstant} . "'"; -} - -# Export functions and constants -our %EXPORT_TAGS = %{pgBackRest::LibCAuto::libcAutoExportTag()}; -our @EXPORT_OK; - -foreach my $strSection (keys(%EXPORT_TAGS)) -{ - # Assign values to serial constants like CFGCMD_* and CFGOPT_*. New commands and options (especially options) renumber the list - # and cause a lot of churn in the commits. This takes care of the renumbering to cut down on that churn. - my $strPrefixLast = 'XXXXXXXX'; - my $iConstantIdx = 0; - - foreach my $strConstant (@{$EXPORT_TAGS{$strSection}}) - { - my $strPrefix = ($strConstant =~ m/^[A-Z0-9]+/g)[0]; - - if (defined($strPrefix)) - { - if ($strPrefix ne $strPrefixLast) - { - $iConstantIdx = 0; - } - else - { - $iConstantIdx++; - } - - if ($strPrefix eq 'CFGCMD' || $strPrefix eq 'CFGOPT') - { - eval "use constant ${strConstant} => ${iConstantIdx}"; - } - - $strPrefixLast = $strPrefix; - } - } - - # OK to export everything in the tag - push(@EXPORT_OK, @{$EXPORT_TAGS{$strSection}}); -} - -# Nothing is exported by default -our @EXPORT = (); - -1; diff --git a/lib/pgBackRest/LibCAuto.pm b/lib/pgBackRest/LibCAuto.pm deleted file mode 100644 index 2cd0531a6..000000000 --- a/lib/pgBackRest/LibCAuto.pm +++ /dev/null @@ -1,356 +0,0 @@ -#################################################################################################################################### -# Automatically generated by Build.pm -- do not modify directly. -#################################################################################################################################### -package pgBackRest::LibCAuto; - -use strict; -use warnings; - -# Configuration option value constants -sub libcAutoConstant -{ - return - { - CFGOPTVAL_COMPRESS_TYPE_NONE => 'none', - CFGOPTVAL_COMPRESS_TYPE_GZ => 'gz', - - CFGOPTVAL_INFO_OUTPUT_TEXT => 'text', - CFGOPTVAL_INFO_OUTPUT_JSON => 'json', - - CFGOPTVAL_REPO_LS_OUTPUT_TEXT => 'text', - CFGOPTVAL_REPO_LS_OUTPUT_JSON => 'json', - - CFGOPTVAL_REMOTE_TYPE_PG => 'pg', - CFGOPTVAL_REMOTE_TYPE_REPO => 'repo', - - CFGOPTVAL_REPO_CIPHER_TYPE_NONE => 'none', - CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc', - - CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_FULL => 'full', - CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_DIFF => 'diff', - CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_INCR => 'incr', - - CFGOPTVAL_REPO_S3_URI_STYLE_HOST => 'host', - CFGOPTVAL_REPO_S3_URI_STYLE_PATH => 'path', - - CFGOPTVAL_REPO_TYPE_CIFS => 'cifs', - CFGOPTVAL_REPO_TYPE_POSIX => 'posix', - CFGOPTVAL_REPO_TYPE_S3 => 's3', - - CFGOPTVAL_SORT_NONE => 'none', - CFGOPTVAL_SORT_ASC => 'asc', - CFGOPTVAL_SORT_DESC => 'desc', - - CFGOPTVAL_RESTORE_TARGET_ACTION_PAUSE => 'pause', - CFGOPTVAL_RESTORE_TARGET_ACTION_PROMOTE => 'promote', - CFGOPTVAL_RESTORE_TARGET_ACTION_SHUTDOWN => 'shutdown', - - CFGOPTVAL_BACKUP_TYPE_FULL => 'full', - CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff', - CFGOPTVAL_BACKUP_TYPE_INCR => 'incr', - - CFGOPTVAL_RESTORE_TYPE_NAME => 'name', - CFGOPTVAL_RESTORE_TYPE_TIME => 'time', - CFGOPTVAL_RESTORE_TYPE_XID => 'xid', - CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve', - CFGOPTVAL_RESTORE_TYPE_NONE => 'none', - CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate', - CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default', - CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby', - - CFGDEF_TYPE_BOOLEAN => 0, - CFGDEF_TYPE_FLOAT => 1, - CFGDEF_TYPE_HASH => 2, - CFGDEF_TYPE_INTEGER => 3, - CFGDEF_TYPE_LIST => 4, - CFGDEF_TYPE_PATH => 5, - CFGDEF_TYPE_SIZE => 6, - CFGDEF_TYPE_STRING => 7, - - ENCODE_TYPE_BASE64 => 0, - - CIPHER_MODE_ENCRYPT => 0, - CIPHER_MODE_DECRYPT => 1, - } -} - -# Export function and constants -sub libcAutoExportTag -{ - return - { - config => - [ - 'CFGOPTVAL_COMPRESS_TYPE_NONE', - 'CFGOPTVAL_COMPRESS_TYPE_GZ', - 'CFGOPTVAL_INFO_OUTPUT_TEXT', - 'CFGOPTVAL_INFO_OUTPUT_JSON', - 'CFGOPTVAL_REPO_LS_OUTPUT_TEXT', - 'CFGOPTVAL_REPO_LS_OUTPUT_JSON', - 'CFGOPTVAL_REMOTE_TYPE_PG', - 'CFGOPTVAL_REMOTE_TYPE_REPO', - 'CFGOPTVAL_REPO_CIPHER_TYPE_NONE', - 'CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC', - 'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_FULL', - 'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_DIFF', - 'CFGOPTVAL_REPO_RETENTION_ARCHIVE_TYPE_INCR', - 'CFGOPTVAL_REPO_S3_URI_STYLE_HOST', - 'CFGOPTVAL_REPO_S3_URI_STYLE_PATH', - 'CFGOPTVAL_REPO_TYPE_CIFS', - 'CFGOPTVAL_REPO_TYPE_POSIX', - 'CFGOPTVAL_REPO_TYPE_S3', - 'CFGOPTVAL_SORT_NONE', - 'CFGOPTVAL_SORT_ASC', - 'CFGOPTVAL_SORT_DESC', - 'CFGOPTVAL_RESTORE_TARGET_ACTION_PAUSE', - 'CFGOPTVAL_RESTORE_TARGET_ACTION_PROMOTE', - 'CFGOPTVAL_RESTORE_TARGET_ACTION_SHUTDOWN', - 'CFGOPTVAL_BACKUP_TYPE_FULL', - 'CFGOPTVAL_BACKUP_TYPE_DIFF', - 'CFGOPTVAL_BACKUP_TYPE_INCR', - 'CFGOPTVAL_RESTORE_TYPE_NAME', - 'CFGOPTVAL_RESTORE_TYPE_TIME', - 'CFGOPTVAL_RESTORE_TYPE_XID', - 'CFGOPTVAL_RESTORE_TYPE_PRESERVE', - 'CFGOPTVAL_RESTORE_TYPE_NONE', - 'CFGOPTVAL_RESTORE_TYPE_IMMEDIATE', - 'CFGOPTVAL_RESTORE_TYPE_DEFAULT', - 'CFGOPTVAL_RESTORE_TYPE_STANDBY', - 'CFGCMD_ARCHIVE_GET', - 'CFGCMD_ARCHIVE_PUSH', - 'CFGCMD_BACKUP', - 'CFGCMD_CHECK', - 'CFGCMD_EXPIRE', - 'CFGCMD_HELP', - 'CFGCMD_INFO', - 'CFGCMD_REPO_CREATE', - 'CFGCMD_REPO_GET', - 'CFGCMD_REPO_LS', - 'CFGCMD_REPO_PUT', - 'CFGCMD_REPO_RM', - 'CFGCMD_RESTORE', - 'CFGCMD_STANZA_CREATE', - 'CFGCMD_STANZA_DELETE', - 'CFGCMD_STANZA_UPGRADE', - 'CFGCMD_START', - 'CFGCMD_STOP', - 'CFGCMD_VERSION', - 'CFGOPT_ARCHIVE_ASYNC', - 'CFGOPT_ARCHIVE_CHECK', - 'CFGOPT_ARCHIVE_COPY', - 'CFGOPT_ARCHIVE_GET_QUEUE_MAX', - 'CFGOPT_ARCHIVE_PUSH_QUEUE_MAX', - 'CFGOPT_ARCHIVE_TIMEOUT', - 'CFGOPT_BACKUP_STANDBY', - 'CFGOPT_BUFFER_SIZE', - 'CFGOPT_CHECKSUM_PAGE', - 'CFGOPT_CIPHER_PASS', - 'CFGOPT_CMD_SSH', - 'CFGOPT_COMPRESS', - 'CFGOPT_COMPRESS_LEVEL', - 'CFGOPT_COMPRESS_LEVEL_NETWORK', - 'CFGOPT_COMPRESS_TYPE', - 'CFGOPT_CONFIG', - 'CFGOPT_CONFIG_INCLUDE_PATH', - 'CFGOPT_CONFIG_PATH', - 'CFGOPT_DB_INCLUDE', - 'CFGOPT_DB_TIMEOUT', - 'CFGOPT_DELTA', - 'CFGOPT_EXCLUDE', - 'CFGOPT_FILTER', - 'CFGOPT_FORCE', - 'CFGOPT_HOST_ID', - 'CFGOPT_IGNORE_MISSING', - 'CFGOPT_LINK_ALL', - 'CFGOPT_LINK_MAP', - 'CFGOPT_LOCK_PATH', - 'CFGOPT_LOG_LEVEL_CONSOLE', - 'CFGOPT_LOG_LEVEL_FILE', - 'CFGOPT_LOG_LEVEL_STDERR', - 'CFGOPT_LOG_PATH', - 'CFGOPT_LOG_SUBPROCESS', - 'CFGOPT_LOG_TIMESTAMP', - 'CFGOPT_MANIFEST_SAVE_THRESHOLD', - 'CFGOPT_NEUTRAL_UMASK', - 'CFGOPT_ONLINE', - 'CFGOPT_OUTPUT', - 'CFGOPT_PG_HOST', - 'CFGOPT_PG_HOST2', - 'CFGOPT_PG_HOST3', - 'CFGOPT_PG_HOST4', - 'CFGOPT_PG_HOST5', - 'CFGOPT_PG_HOST6', - 'CFGOPT_PG_HOST7', - 'CFGOPT_PG_HOST8', - 'CFGOPT_PG_HOST_CMD', - 'CFGOPT_PG_HOST_CMD2', - 'CFGOPT_PG_HOST_CMD3', - 'CFGOPT_PG_HOST_CMD4', - 'CFGOPT_PG_HOST_CMD5', - 'CFGOPT_PG_HOST_CMD6', - 'CFGOPT_PG_HOST_CMD7', - 'CFGOPT_PG_HOST_CMD8', - 'CFGOPT_PG_HOST_CONFIG', - 'CFGOPT_PG_HOST_CONFIG2', - 'CFGOPT_PG_HOST_CONFIG3', - 'CFGOPT_PG_HOST_CONFIG4', - 'CFGOPT_PG_HOST_CONFIG5', - 'CFGOPT_PG_HOST_CONFIG6', - 'CFGOPT_PG_HOST_CONFIG7', - 'CFGOPT_PG_HOST_CONFIG8', - 'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH', - 'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH2', - 'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH3', - 'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH4', - 'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH5', - 'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH6', - 'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH7', - 'CFGOPT_PG_HOST_CONFIG_INCLUDE_PATH8', - 'CFGOPT_PG_HOST_CONFIG_PATH', - 'CFGOPT_PG_HOST_CONFIG_PATH2', - 'CFGOPT_PG_HOST_CONFIG_PATH3', - 'CFGOPT_PG_HOST_CONFIG_PATH4', - 'CFGOPT_PG_HOST_CONFIG_PATH5', - 'CFGOPT_PG_HOST_CONFIG_PATH6', - 'CFGOPT_PG_HOST_CONFIG_PATH7', - 'CFGOPT_PG_HOST_CONFIG_PATH8', - 'CFGOPT_PG_HOST_PORT', - 'CFGOPT_PG_HOST_PORT2', - 'CFGOPT_PG_HOST_PORT3', - 'CFGOPT_PG_HOST_PORT4', - 'CFGOPT_PG_HOST_PORT5', - 'CFGOPT_PG_HOST_PORT6', - 'CFGOPT_PG_HOST_PORT7', - 'CFGOPT_PG_HOST_PORT8', - 'CFGOPT_PG_HOST_USER', - 'CFGOPT_PG_HOST_USER2', - 'CFGOPT_PG_HOST_USER3', - 'CFGOPT_PG_HOST_USER4', - 'CFGOPT_PG_HOST_USER5', - 'CFGOPT_PG_HOST_USER6', - 'CFGOPT_PG_HOST_USER7', - 'CFGOPT_PG_HOST_USER8', - 'CFGOPT_PG_PATH', - 'CFGOPT_PG_PATH2', - 'CFGOPT_PG_PATH3', - 'CFGOPT_PG_PATH4', - 'CFGOPT_PG_PATH5', - 'CFGOPT_PG_PATH6', - 'CFGOPT_PG_PATH7', - 'CFGOPT_PG_PATH8', - 'CFGOPT_PG_PORT', - 'CFGOPT_PG_PORT2', - 'CFGOPT_PG_PORT3', - 'CFGOPT_PG_PORT4', - 'CFGOPT_PG_PORT5', - 'CFGOPT_PG_PORT6', - 'CFGOPT_PG_PORT7', - 'CFGOPT_PG_PORT8', - 'CFGOPT_PG_SOCKET_PATH', - 'CFGOPT_PG_SOCKET_PATH2', - 'CFGOPT_PG_SOCKET_PATH3', - 'CFGOPT_PG_SOCKET_PATH4', - 'CFGOPT_PG_SOCKET_PATH5', - 'CFGOPT_PG_SOCKET_PATH6', - 'CFGOPT_PG_SOCKET_PATH7', - 'CFGOPT_PG_SOCKET_PATH8', - 'CFGOPT_PG_USER', - 'CFGOPT_PG_USER2', - 'CFGOPT_PG_USER3', - 'CFGOPT_PG_USER4', - 'CFGOPT_PG_USER5', - 'CFGOPT_PG_USER6', - 'CFGOPT_PG_USER7', - 'CFGOPT_PG_USER8', - 'CFGOPT_PROCESS', - 'CFGOPT_PROCESS_MAX', - 'CFGOPT_PROTOCOL_TIMEOUT', - 'CFGOPT_RAW', - 'CFGOPT_RECOVERY_OPTION', - 'CFGOPT_RECURSE', - 'CFGOPT_REMOTE_TYPE', - 'CFGOPT_REPO_CIPHER_PASS', - 'CFGOPT_REPO_CIPHER_TYPE', - 'CFGOPT_REPO_HARDLINK', - 'CFGOPT_REPO_HOST', - 'CFGOPT_REPO_HOST_CMD', - 'CFGOPT_REPO_HOST_CONFIG', - 'CFGOPT_REPO_HOST_CONFIG_INCLUDE_PATH', - 'CFGOPT_REPO_HOST_CONFIG_PATH', - 'CFGOPT_REPO_HOST_PORT', - 'CFGOPT_REPO_HOST_USER', - 'CFGOPT_REPO_PATH', - 'CFGOPT_REPO_RETENTION_ARCHIVE', - 'CFGOPT_REPO_RETENTION_ARCHIVE_TYPE', - 'CFGOPT_REPO_RETENTION_DIFF', - 'CFGOPT_REPO_RETENTION_FULL', - 'CFGOPT_REPO_S3_BUCKET', - 'CFGOPT_REPO_S3_CA_FILE', - 'CFGOPT_REPO_S3_CA_PATH', - 'CFGOPT_REPO_S3_ENDPOINT', - 'CFGOPT_REPO_S3_HOST', - 'CFGOPT_REPO_S3_KEY', - 'CFGOPT_REPO_S3_KEY_SECRET', - 'CFGOPT_REPO_S3_PORT', - 'CFGOPT_REPO_S3_REGION', - 'CFGOPT_REPO_S3_TOKEN', - 'CFGOPT_REPO_S3_URI_STYLE', - 'CFGOPT_REPO_S3_VERIFY_TLS', - 'CFGOPT_REPO_TYPE', - 'CFGOPT_RESUME', - 'CFGOPT_SET', - 'CFGOPT_SORT', - 'CFGOPT_SPOOL_PATH', - 'CFGOPT_STANZA', - 'CFGOPT_START_FAST', - 'CFGOPT_STOP_AUTO', - 'CFGOPT_TABLESPACE_MAP', - 'CFGOPT_TABLESPACE_MAP_ALL', - 'CFGOPT_TARGET', - 'CFGOPT_TARGET_ACTION', - 'CFGOPT_TARGET_EXCLUSIVE', - 'CFGOPT_TARGET_TIMELINE', - 'CFGOPT_TYPE', - 'cfgCommandName', - 'cfgOptionIndexTotal', - 'cfgOptionName', - ], - - configDefine => - [ - 'CFGDEF_TYPE_BOOLEAN', - 'CFGDEF_TYPE_FLOAT', - 'CFGDEF_TYPE_HASH', - 'CFGDEF_TYPE_INTEGER', - 'CFGDEF_TYPE_LIST', - 'CFGDEF_TYPE_PATH', - 'CFGDEF_TYPE_SIZE', - 'CFGDEF_TYPE_STRING', - 'cfgCommandId', - 'cfgDefOptionDefault', - 'cfgDefOptionPrefix', - 'cfgDefOptionSecure', - 'cfgDefOptionType', - 'cfgDefOptionValid', - 'cfgOptionId', - 'cfgOptionTotal', - ], - - debug => - [ - 'libcUvSize', - ], - - storage => - [ - 'storageRepoFree', - ], - - test => - [ - 'cfgParseTest', - ], - } -} - -1; diff --git a/lib/pgBackRest/Manifest.pm b/lib/pgBackRest/Manifest.pm index d48fe0a35..4199f1052 100644 --- a/lib/pgBackRest/Manifest.pm +++ b/lib/pgBackRest/Manifest.pm @@ -18,8 +18,6 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; #################################################################################################################################### @@ -94,31 +92,31 @@ use constant MANIFEST_KEY_TYPE => 'backup-t push @EXPORT, qw(MANIFEST_KEY_TYPE); # Options that were set when the backup was made -use constant MANIFEST_KEY_BACKUP_STANDBY => 'option-' . cfgOptionName(CFGOPT_BACKUP_STANDBY); +use constant MANIFEST_KEY_BACKUP_STANDBY => 'option-backup-standby'; push @EXPORT, qw(MANIFEST_KEY_BACKUP_STANDBY); use constant MANIFEST_KEY_HARDLINK => 'option-hardlink'; push @EXPORT, qw(MANIFEST_KEY_HARDLINK); -use constant MANIFEST_KEY_ARCHIVE_CHECK => 'option-' . cfgOptionName(CFGOPT_ARCHIVE_CHECK); +use constant MANIFEST_KEY_ARCHIVE_CHECK => 'option-archive-check'; push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_CHECK); -use constant MANIFEST_KEY_ARCHIVE_COPY => 'option-' .cfgOptionName(CFGOPT_ARCHIVE_COPY); +use constant MANIFEST_KEY_ARCHIVE_COPY => 'option-archive-copy'; push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_COPY); -use constant MANIFEST_KEY_BUFFER_SIZE => 'option-' . cfgOptionName(CFGOPT_BUFFER_SIZE); +use constant MANIFEST_KEY_BUFFER_SIZE => 'option-buffer-size'; push @EXPORT, qw(MANIFEST_KEY_BUFFER_SIZE); -use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-' . cfgOptionName(CFGOPT_CHECKSUM_PAGE); +use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-checksum-page'; push @EXPORT, qw(MANIFEST_KEY_CHECKSUM_PAGE); -use constant MANIFEST_KEY_COMPRESS => 'option-' . cfgOptionName(CFGOPT_COMPRESS); +use constant MANIFEST_KEY_COMPRESS => 'option-compress'; push @EXPORT, qw(MANIFEST_KEY_COMPRESS); -use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-' . cfgOptionName(CFGOPT_COMPRESS_TYPE); +use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-compress-type'; push @EXPORT, qw(MANIFEST_KEY_COMPRESS_TYPE); -use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-' . cfgOptionName(CFGOPT_COMPRESS_LEVEL); +use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-compress-level'; push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL); -use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-' . cfgOptionName(CFGOPT_COMPRESS_LEVEL_NETWORK); +use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-compress-level-network'; push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL_NETWORK); -use constant MANIFEST_KEY_ONLINE => 'option-' . cfgOptionName(CFGOPT_ONLINE); +use constant MANIFEST_KEY_ONLINE => 'option-online'; push @EXPORT, qw(MANIFEST_KEY_ONLINE); -use constant MANIFEST_KEY_DELTA => 'option-' . cfgOptionName(CFGOPT_DELTA); +use constant MANIFEST_KEY_DELTA => 'option-delta'; push @EXPORT, qw(MANIFEST_KEY_DELTA); -use constant MANIFEST_KEY_PROCESS_MAX => 'option-' . cfgOptionName(CFGOPT_PROCESS_MAX); +use constant MANIFEST_KEY_PROCESS_MAX => 'option-process-max'; push @EXPORT, qw(MANIFEST_KEY_PROCESS_MAX); # Information about the database that was backed up @@ -320,8 +318,8 @@ sub new ); # Init object and store variables - my $self = $class->SUPER::new($strFileName, {bLoad => $bLoad, oStorage => $oStorage, strCipherPass => $strCipherPass, - strCipherPassSub => $strCipherPassSub}); + my $self = $class->SUPER::new( + $oStorage, $strFileName, {bLoad => $bLoad, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub}); # If manifest not loaded from a file then the db version and catalog version must be set if (!$bLoad) @@ -1111,9 +1109,6 @@ sub build # consistent anyway and the one-second resolution problem is the least of our worries). my $lTimeBegin = waitRemainder($bOnline); - # Check that links are valid - $self->linkCheck(); - if (defined($oLastManifest)) { $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, @@ -1244,52 +1239,6 @@ sub build ); } -#################################################################################################################################### -# linkCheck -# -# Check all link targets and make sure none of them are a subset of another link. In theory it would be possible to resolve the -# dependencies and generate a valid backup/restore but it's really complicated and there don't seem to be any compelling use cases. -#################################################################################################################################### -sub linkCheck -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->linkCheck'); - - # Working variable - my $strBasePath = $self->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); - - foreach my $strTargetParent ($self->keys(MANIFEST_SECTION_BACKUP_TARGET)) - { - if ($self->isTargetLink($strTargetParent)) - { - my $strParentPath = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetParent, MANIFEST_SUBKEY_PATH); - my $strParentFile = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetParent, MANIFEST_SUBKEY_FILE, false); - - foreach my $strTargetChild ($self->keys(MANIFEST_SECTION_BACKUP_TARGET)) - { - if ($self->isTargetLink($strTargetChild) && $strTargetParent ne $strTargetChild) - { - my $strChildPath = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetChild, MANIFEST_SUBKEY_PATH); - my $strChildFile = $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strTargetParent, MANIFEST_SUBKEY_FILE, false); - - if (!(defined($strParentFile) && defined($strChildFile)) && - index( - storageLocal()->pathAbsolute($strBasePath, $strChildPath) . '/', - storageLocal()->pathAbsolute($strBasePath, $strParentPath) . '/') == 0) - { - confess &log(ERROR, 'link ' . $self->dbPathGet($strBasePath, $strTargetChild) . - " (${strChildPath}) references a subdirectory of or" . - " the same directory as link " . $self->dbPathGet($strBasePath, $strTargetParent) . - " (${strParentPath})", ERROR_LINK_DESTINATION); - } - } - } - } - } -} - #################################################################################################################################### # fileAdd # diff --git a/lib/pgBackRest/Protocol/Storage/Helper.pm b/lib/pgBackRest/Protocol/Storage/Helper.pm deleted file mode 100644 index 7eb23be03..000000000 --- a/lib/pgBackRest/Protocol/Storage/Helper.pm +++ /dev/null @@ -1,89 +0,0 @@ -#################################################################################################################################### -# Db & Repository Storage Helper -#################################################################################################################################### -package pgBackRest::Protocol::Storage::Helper; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(basename); - -use pgBackRest::Common::Log; -use pgBackRest::Config::Config; -use pgBackRest::LibC qw(:storage); -use pgBackRest::Storage::Helper; - -#################################################################################################################################### -# Storage constants -#################################################################################################################################### -use constant STORAGE_DB => ''; - push @EXPORT, qw(STORAGE_DB); - -use constant STORAGE_REPO => ''; - push @EXPORT, qw(STORAGE_REPO); -use constant STORAGE_REPO_ARCHIVE => ''; - push @EXPORT, qw(STORAGE_REPO_ARCHIVE); -use constant STORAGE_REPO_BACKUP => ''; - push @EXPORT, qw(STORAGE_REPO_BACKUP); - -#################################################################################################################################### -# Cache storage so it can be retrieved quickly -#################################################################################################################################### -my $hStorage; - -#################################################################################################################################### -# storageRepo - get repository storage -#################################################################################################################################### -sub storageRepo -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - ) = - logDebugParam - ( - __PACKAGE__ . '::storageRepo', \@_, - {name => 'strStanza', optional => true, trace => true}, - ); - - # Create storage if not defined - if (!defined($hStorage->{&STORAGE_REPO})) - { - $hStorage->{&STORAGE_REPO} = new pgBackRest::Storage::Storage( - STORAGE_REPO, {lBufferMax => cfgOption(CFGOPT_BUFFER_SIZE)}); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oStorageRepo', value => $hStorage->{&STORAGE_REPO}, trace => true}, - ); -} - -push @EXPORT, qw(storageRepo); - -#################################################################################################################################### -# Clear the repo storage cache - FOR TESTING ONLY! -#################################################################################################################################### -sub storageRepoCacheClear -{ - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageRepoCacheClear'); - - delete($hStorage->{&STORAGE_REPO}); - - storageRepoFree(); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push @EXPORT, qw(storageRepoCacheClear); - -1; diff --git a/lib/pgBackRest/Storage/Base.pm b/lib/pgBackRest/Storage/Base.pm index 9f9b29f13..0da8f61cd 100644 --- a/lib/pgBackRest/Storage/Base.pm +++ b/lib/pgBackRest/Storage/Base.pm @@ -23,8 +23,8 @@ use pgBackRest::Common::Log; use constant STORAGE_LOCAL => ''; push @EXPORT, qw(STORAGE_LOCAL); -use constant STORAGE_S3 => 's3'; - push @EXPORT, qw(STORAGE_S3); +use constant STORAGE_OBJECT => 'object'; + push @EXPORT, qw(STORAGE_OBJECT); use constant STORAGE_POSIX => 'posix'; push @EXPORT, qw(STORAGE_POSIX); @@ -71,145 +71,6 @@ sub new ); } - -#################################################################################################################################### -# Copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed instead of -# file names. -#################################################################################################################################### -sub copy -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $xSourceFile, - $xDestinationFile, - $bSourceOpen, - ) = - logDebugParam - ( - __PACKAGE__ . '->copy', \@_, - {name => 'xSourceFile', required => false}, - {name => 'xDestinationFile'}, - {name => 'bSourceOpen', optional => true, default => false}, - ); - - # Is source/destination an IO object or a file expression? - my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef; - - # Does the source file exist? - my $bResult = false; - - # Copy if the source file exists - if (defined($oSourceFileIo)) - { - my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile); - - # Use C copy if source and destination are C objects - if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite})) - { - $bResult = $self->{oStorageC}->copy( - $oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false; - } - else - { - # Open the source file if it is a C object - $bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true; - - if ($bResult) - { - # Open the destination file if it is a C object - if (defined($oDestinationFileIo->{oStorageCWrite})) - { - $oDestinationFileIo->open(); - } - - # Copy the data - do - { - # Read data - my $tBuffer = ''; - - $oSourceFileIo->read(\$tBuffer, $self->{lBufferMax}); - $oDestinationFileIo->write(\$tBuffer); - } - while (!$oSourceFileIo->eof()); - - # Close files - $oSourceFileIo->close(); - $oDestinationFileIo->close(); - } - } - } - - return logDebugReturn - ( - $strOperation, - {name => 'bResult', value => $bResult, trace => true}, - ); -} - -#################################################################################################################################### -# get - reads a buffer from storage all at once -#################################################################################################################################### -sub get -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $xFile, - $strCipherPass, - ) = - logDebugParam - ( - __PACKAGE__ . '->get', \@_, - {name => 'xFile', required => false, trace => true}, - {name => 'strCipherPass', optional => true, redact => true}, - ); - - # Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or - # if the repo has a user passphrase defined - else pass undef - my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead( - $xFile, {strCipherPass => defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()})) : undef; - - # Read only if there is something to read from - my $tContent; - my $lSize = 0; - - if (defined($oFileIo)) - { - my $lSizeRead; - - do - { - $lSizeRead = $oFileIo->read(\$tContent, $self->{lBufferMax}); - $lSize += $lSizeRead; - } - while ($lSizeRead != 0); - - # Close the file - $oFileIo->close(); - - # If nothing was read then set to undef - if ($lSize == 0) - { - $tContent = undef; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rtContent', value => defined($oFileIo) ? \$tContent : undef, trace => true}, - ); -} - #################################################################################################################################### # Calculate sha1 hash and size of file. If special encryption settings are required, then the file objects from openRead/openWrite # must be passed instead of file names. @@ -328,57 +189,4 @@ sub pathAbsolute ); } -#################################################################################################################################### -# put - writes a buffer out to storage all at once -#################################################################################################################################### -sub put -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $xFile, - $xContent, - $strCipherPass, - ) = - logDebugParam - ( - __PACKAGE__ . '->put', \@_, - {name => 'xFile', trace => true}, - {name => 'xContent', required => false, trace => true}, - {name => 'strCipherPass', optional => true, trace => true, redact => true}, - ); - - # Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if - # the repo has a user passphrase defined - else pass undef - my $oFileIo = ref($xFile) ? $xFile : $self->openWrite( - $xFile, {strCipherPass => defined($strCipherPass) ? $strCipherPass : $self->cipherPassUser()}); - - # Determine size of content - my $lSize = defined($xContent) ? length(ref($xContent) ? $$xContent : $xContent) : 0; - - # Write only if there is something to write - if ($lSize > 0) - { - $oFileIo->write(ref($xContent) ? $xContent : \$xContent); - } - # Else open the file so a zero length file is created (since file is not opened until first write) - else - { - $oFileIo->open(); - } - - # Close the file - $oFileIo->close(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'lSize', value => $lSize, trace => true}, - ); -} - 1; diff --git a/lib/pgBackRest/Storage/Helper.pm b/lib/pgBackRest/Storage/Helper.pm index 05c083339..ae1c192df 100644 --- a/lib/pgBackRest/Storage/Helper.pm +++ b/lib/pgBackRest/Storage/Helper.pm @@ -1,5 +1,5 @@ #################################################################################################################################### -# Local Storage Helper +# Repository Storage Helper #################################################################################################################################### package pgBackRest::Storage::Helper; @@ -12,17 +12,9 @@ use Exporter qw(import); use File::Basename qw(basename); use pgBackRest::Common::Log; -use pgBackRest::Config::Config; -use pgBackRest::Storage::Base; use pgBackRest::Storage::Storage; use pgBackRest::Version; -#################################################################################################################################### -# Compression extension -#################################################################################################################################### -use constant COMPRESS_EXT => 'gz'; - push @EXPORT, qw(COMPRESS_EXT); - #################################################################################################################################### # Temp file extension #################################################################################################################################### @@ -30,24 +22,88 @@ use constant STORAGE_TEMP_EXT => PROJECT_E push @EXPORT, qw(STORAGE_TEMP_EXT); #################################################################################################################################### -# storageLocal - get local storage -# -# Local storage is generally read-only (except for locking) and can never reference a remote path. Used for adhoc activities like -# reading pgbackrest.conf. +# Cache storage so it can be retrieved quickly #################################################################################################################################### -sub storageLocal +my $oRepoStorage; + +#################################################################################################################################### +# storageRepoCommandSet +#################################################################################################################################### +my $strStorageRepoCommand; +my $strStorageRepoType; + +sub storageRepoCommandSet { # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageLocal'); + my + ( + $strOperation, + $strCommand, + $strStorageType, + ) = + logDebugParam + ( + __PACKAGE__ . '::storageRepoCommandSet', \@_, + {name => 'strCommand'}, + {name => 'strStorageType'}, + ); + + $strStorageRepoCommand = $strCommand; + $strStorageRepoType = $strStorageType; + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +push @EXPORT, qw(storageRepoCommandSet); + +#################################################################################################################################### +# storageRepo - get repository storage +#################################################################################################################################### +sub storageRepo +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strStanza, + ) = + logDebugParam + ( + __PACKAGE__ . '::storageRepo', \@_, + {name => 'strStanza', optional => true, trace => true}, + ); + + # Create storage if not defined + if (!defined($oRepoStorage)) + { + $oRepoStorage = new pgBackRest::Storage::Storage($strStorageRepoCommand, $strStorageRepoType, 64 * 1024, 60); + } # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'oStorageLocal', value => new pgBackRest::Storage::Storage(STORAGE_LOCAL), trace => true}, + {name => 'oStorageRepo', value => $oRepoStorage, trace => true}, ); } -push @EXPORT, qw(storageLocal); +push @EXPORT, qw(storageRepo); + +#################################################################################################################################### +# Clear the repo storage cache - FOR TESTING ONLY! +#################################################################################################################################### +sub storageRepoCacheClear +{ + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '::storageRepoCacheClear'); + + undef($oRepoStorage); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +push @EXPORT, qw(storageRepoCacheClear); 1; diff --git a/lib/pgBackRest/Storage/Storage.pm b/lib/pgBackRest/Storage/Storage.pm index d5280574e..91de58a7d 100644 --- a/lib/pgBackRest/Storage/Storage.pm +++ b/lib/pgBackRest/Storage/Storage.pm @@ -17,10 +17,9 @@ use JSON::PP; use pgBackRest::Common::Exception; use pgBackRest::Common::Io::Handle; +use pgBackRest::Common::Io::Process; use pgBackRest::Common::Log; use pgBackRest::Storage::Base; -use pgBackRest::Storage::StorageRead; -use pgBackRest::Storage::StorageWrite; #################################################################################################################################### # new @@ -36,32 +35,24 @@ sub new # Assign function parameters, defaults, and log debug info ( my $strOperation, + $self->{strCommand}, $self->{strType}, - $self->{strPath}, $self->{lBufferMax}, + $self->{iTimeoutIo}, $self->{strDefaultPathMode}, $self->{strDefaultFileMode}, ) = logDebugParam ( __PACKAGE__ . '->new', \@_, + {name => 'strCommand'}, {name => 'strType'}, - {name => 'strPath', optional => true}, - {name => 'lBufferMax', optional => true, default => 65536}, + {name => 'lBufferMax'}, + {name => 'iTimeoutIo'}, {name => 'strDefaultPathMode', optional => true, default => '0750'}, {name => 'strDefaultFileMode', optional => true, default => '0640'}, ); - # Create C storage object - $self->{oStorageC} = pgBackRest::LibC::Storage->new($self->{strType}, $self->{strPath}); - - # Get encryption settings - if ($self->{strType} eq '') - { - $self->{strCipherType} = $self->{oStorageC}->cipherType(); - $self->{strCipherPass} = $self->{oStorageC}->cipherPass(); - } - # Create JSON object $self->{oJSON} = JSON::PP->new()->allow_nonref(); @@ -73,6 +64,103 @@ sub new ); } +#################################################################################################################################### +# Escape characteres that have special meaning on the command line +#################################################################################################################################### +sub escape +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strValue, + ) = + logDebugParam + ( + __PACKAGE__ . '->escape', \@_, + {name => 'strValue', trace => true}, + ); + + $strValue =~ s/\\/\\\\/g; + $strValue =~ s/\/\\\>/g; + $strValue =~ s/\!/\\\!/g; + $strValue =~ s/\*/\\\*/g; + $strValue =~ s/\(/\\\(/g; + $strValue =~ s/\)/\\\)/g; + $strValue =~ s/\&/\\\&/g; + $strValue =~ s/\'/\\\'/g; + $strValue =~ s/\;/\\\;/g; + $strValue =~ s/\?/\\\?/g; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strValue', value => $strValue}, + ); +} + +#################################################################################################################################### +# Execute command and return the output +#################################################################################################################################### +sub exec +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strCommand, + ) = + logDebugParam + ( + __PACKAGE__ . '->exec', \@_, + {name => 'strCommand'}, + ); + + $strCommand = "$self->{strCommand} ${strCommand}"; + my $oBuffer = new pgBackRest::Common::Io::Buffered( + new pgBackRest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax}); + my $oProcess = new pgBackRest::Common::Io::Process($oBuffer, $strCommand); + + my $tResult; + + while (!$oBuffer->eof()) + { + $oBuffer->read(\$tResult, $self->{lBufferMax}, false); + } + + $oProcess->close(); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'tResult', value => $tResult}, + {name => 'iExitStatus', value => $oProcess->exitStatus()}, + ); +} + +#################################################################################################################################### +# Create storage +#################################################################################################################################### +sub create +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my ($strOperation) = logDebugParam(__PACKAGE__ . '->create'); + + $self->exec("repo-create"); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + #################################################################################################################################### # Check if file exists (not a path) #################################################################################################################################### @@ -96,7 +184,7 @@ sub exists return logDebugReturn ( $strOperation, - {name => 'bExists', value => defined($self->info($strFileExp, {bIgnoreMissing => true}))} + {name => 'bExists', value => $self->info($strFileExp, {bIgnoreMissing => true})->{type} eq 'f'} ); } @@ -113,33 +201,49 @@ sub get $strOperation, $xFile, $strCipherPass, + $bRaw, ) = logDebugParam ( __PACKAGE__ . '->get', \@_, - {name => 'xFile', required => false, trace => true}, - {name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true}, + {name => 'xFile', required => false}, + {name => 'strCipherPass', optional => true, redact => true}, + {name => 'bRaw', optional => true, default => false}, ); - # Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or - # if the repo has a user passphrase defined - else pass undef - my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead($xFile, {strCipherPass => $strCipherPass})) : undef; + # If openRead() was called first set values from that call + my $strFile = $xFile; + my $bIgnoreMissing = false; - # Get the file contents - my $bEmpty = false; - my $tContent = $self->{oStorageC}->get($oFileIo->{oStorageCRead}); - - if (defined($tContent) && length($tContent) == 0) + if (ref($xFile)) { - $tContent = undef; - $bEmpty = true; + $strFile = $xFile->{strFile}; + $bIgnoreMissing = $xFile->{bIgnoreMissing}; + $strCipherPass = $xFile->{strCipherPass}; + } + + # Check invalid params + if ($bRaw && defined($strCipherPass)) + { + confess &log(ERROR, 'bRaw and strCipherPass cannot both be set'); + } + + # Get file + my ($tResult, $iExitStatus) = $self->exec( + (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . ($bRaw ? ' --raw' : '') . + ($bIgnoreMissing ? ' --ignore-missing' : '') . ' repo-get ' . $self->escape($strFile)); + + # Error if missing an not ignored + if ($iExitStatus == 1 && !$bIgnoreMissing) + { + confess &log(ERROR, "unable to open '${strFile}'", ERROR_FILE_OPEN); } # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'rtContent', value => defined($tContent) || $bEmpty ? \$tContent : undef, trace => true}, + {name => 'rtContent', value => $iExitStatus == 0 ? \$tResult : undef, trace => true}, ); } @@ -164,19 +268,11 @@ sub info {name => 'bIgnoreMissing', optional => true, default => false}, ); - my $rhInfo; - my $strJson = $self->{oStorageC}->info($strPathFileExp, $bIgnoreMissing); - - if (defined($strJson)) - { - $rhInfo = $self->{oJSON}->decode($strJson); - } - # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'rhInfo', value => $rhInfo, trace => true} + {name => 'rhInfo', value => $self->manifest($strPathFileExp, {bRecurse => false})->{'.'}, trace => true} ); } @@ -207,11 +303,14 @@ sub list # Get file list my $rstryFileList = []; - my $strFileList = $self->{oStorageC}->list($strPathExp, $bIgnoreMissing, $strSortOrder eq 'forward', $strExpression); + my $rhManifest = $self->manifest($strPathExp, {bRecurse => false}); - if (defined($strFileList) && $strFileList ne '[]') + foreach my $strKey ($strSortOrder eq 'reverse' ? sort {$b cmp $a} keys(%{$rhManifest}) : sort keys(%{$rhManifest})) { - $rstryFileList = $self->{oJSON}->decode($strFileList); + next if $strKey eq '.'; + next if defined($strExpression) && $strKey !~ $strExpression; + + push(@{$rstryFileList}, $strKey); } # Return from function and log return values if any @@ -234,50 +333,54 @@ sub manifest ( $strOperation, $strPathExp, - $strFilter, + $bRecurse, ) = logDebugParam ( __PACKAGE__ . '->manifest', \@_, {name => 'strPathExp'}, - {name => 'strFilter', optional => true, trace => true}, + {name => 'bRecurse', optional => true, default => true}, ); - my $hManifest = $self->{oJSON}->decode($self->manifestJson($strPathExp, {strFilter => $strFilter})); + my $rhManifest = $self->{oJSON}->decode( + $self->exec("--output=json " . ($bRecurse ? ' --recurse' : '') . " repo-ls " . $self->escape($strPathExp))); + + # Transform the manifest to the old format + foreach my $strKey (keys(%{$rhManifest})) + { + if ($rhManifest->{$strKey}{type} eq 'file') + { + $rhManifest->{$strKey}{type} = 'f'; + + if (defined($rhManifest->{$strKey}{time})) + { + $rhManifest->{$strKey}{modified_time} = $rhManifest->{$strKey}{time}; + delete($rhManifest->{$strKey}{time}); + } + } + elsif ($rhManifest->{$strKey}{type} eq 'path') + { + $rhManifest->{$strKey}{type} = 'd'; + } + elsif ($rhManifest->{$strKey}{type} eq 'link') + { + $rhManifest->{$strKey}{type} = 'l'; + } + elsif ($rhManifest->{$strKey}{type} eq 'special') + { + $rhManifest->{$strKey}{type} = 's'; + } + else + { + confess "invalid file type '$rhManifest->{type}'"; + } + } # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'hManifest', value => $hManifest, trace => true} - ); -} - -sub manifestJson -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathExp, - $strFilter, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestJson', \@_, - {name => 'strPathExp'}, - {name => 'strFilter', optional => true, trace => true}, - ); - - my $strManifestJson = $self->{oStorageC}->manifest($strPathExp, $strFilter); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strManifestJson', value => $strManifestJson, trace => true} + {name => 'rhManifest', value => $rhManifest, trace => true} ); } @@ -292,137 +395,24 @@ sub openRead my ( $strOperation, - $xFileExp, + $strFile, $bIgnoreMissing, - $rhyFilter, $strCipherPass, ) = logDebugParam ( __PACKAGE__ . '->openRead', \@_, - {name => 'xFileExp'}, + {name => 'strFile'}, {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'rhyFilter', optional => true}, - {name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true}, + {name => 'strCipherPass', optional => true, redact => true}, ); - # Open the file - my $oFileIo = pgBackRest::LibC::StorageRead->new($self->{oStorageC}, $xFileExp, $bIgnoreMissing); - - # If cipher is set then decryption is the first filter applied to the read - if (defined($self->cipherType())) - { - $oFileIo->filterAdd(STORAGE_FILTER_CIPHER_BLOCK, $self->{oJSON}->encode([false, $self->cipherType(), $strCipherPass])); - } - - # Apply any other filters - if (defined($rhyFilter)) - { - foreach my $rhFilter (@{$rhyFilter}) - { - $oFileIo->filterAdd( - $rhFilter->{strClass}, defined($rhFilter->{rxyParam}) ? $self->{oJSON}->encode($rhFilter->{rxyParam}) : undef); - } - } - # Return from function and log return values if any return logDebugReturn ( $strOperation, - {name => 'oFileIo', value => new pgBackRest::Storage::StorageRead($self, $oFileIo), trace => true}, - ); -} - -#################################################################################################################################### -# Open file for writing -#################################################################################################################################### -sub openWrite -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $xFileExp, - $strMode, - $strUser, - $strGroup, - $lTimestamp, - $bAtomic, - $bPathCreate, - $rhyFilter, - $strCipherPass, - ) = - logDebugParam - ( - __PACKAGE__ . '->openWrite', \@_, - {name => 'xFileExp'}, - {name => 'strMode', optional => true, default => $self->{strDefaultFileMode}}, - {name => 'strUser', optional => true}, - {name => 'strGroup', optional => true}, - {name => 'lTimestamp', optional => true, default => '0'}, - {name => 'bAtomic', optional => true, default => false}, - {name => 'bPathCreate', optional => true, default => true}, - {name => 'rhyFilter', optional => true}, - {name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), redact => true}, - ); - - # Open the file - my $oFileIo = pgBackRest::LibC::StorageWrite->new( - $self->{oStorageC}, $xFileExp, oct($strMode), $strUser, $strGroup, $lTimestamp, $bAtomic, $bPathCreate); - - # Apply any other filters - if (defined($rhyFilter)) - { - foreach my $rhFilter (@{$rhyFilter}) - { - $oFileIo->filterAdd( - $rhFilter->{strClass}, defined($rhFilter->{rxyParam}) ? $self->{oJSON}->encode($rhFilter->{rxyParam}) : undef); - } - } - - # If cipher is set then encryption is the last filter applied to the write - if (defined($self->cipherType())) - { - $oFileIo->filterAdd(STORAGE_FILTER_CIPHER_BLOCK, $self->{oJSON}->encode([true, $self->cipherType(), $strCipherPass])); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oFileIo', value => new pgBackRest::Storage::StorageWrite($self, $oFileIo), trace => true}, - ); -} - -#################################################################################################################################### -# Resolve a path expression into an absolute path -#################################################################################################################################### -sub pathGet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathExp, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathGet', \@_, - {name => 'strPathExp'}, - ); - - # Check exists - my $strPath = $self->{oStorageC}->pathGet($strPathExp); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strPath', value => $strPath, trace => true} + {name => 'rhFileIo', value => {strFile => $strFile, bIgnoreMissing => $bIgnoreMissing, strCipherPass => $strCipherPass}, + trace => true}, ); } @@ -437,19 +427,17 @@ sub pathRemove my ( $strOperation, - $strPathExp, - $bIgnoreMissing, + $strPath, $bRecurse, ) = logDebugParam ( __PACKAGE__ . '->pathRemove', \@_, - {name => 'strPathExp'}, - {name => 'bIgnoreMissing', optional => true, default => true}, + {name => 'strPath'}, {name => 'bRecurse', optional => true, default => false}, ); - $self->{oStorageC}->pathRemove($strPathExp, $bIgnoreMissing, $bRecurse); + $self->exec("repo-rm " . ($bRecurse ? '--recurse ' : '') . $self->escape($strPath)); # Return from function and log return values if any return logDebugReturn($strOperation); @@ -466,31 +454,54 @@ sub put my ( $strOperation, - $xFile, - $xContent, + $strFile, + $tContent, $strCipherPass, + $bRaw, ) = logDebugParam ( __PACKAGE__ . '->put', \@_, - {name => 'xFile', trace => true}, - {name => 'xContent', required => false, trace => true}, - {name => 'strCipherPass', optional => true, default => $self->cipherPassUser(), trace => true, redact => true}, + {name => 'strFile'}, + {name => 'tContent', required => false}, + {name => 'strCipherPass', optional => true, redact => true}, + {name => 'bRaw', optional => true, default => false}, ); - # Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if - # the repo has a user passphrase defined - else pass undef - my $oFileIo = ref($xFile) ? $xFile : $self->openWrite($xFile, {strCipherPass => $strCipherPass}); + # Check invalid params + if ($bRaw && defined($strCipherPass)) + { + confess &log(ERROR, 'bRaw and strCipherPass cannot both be set'); + } - # Write the content - my $lSize = $self->{oStorageC}->put($oFileIo->{oStorageCWrite}, ref($xContent) ? $$xContent : $xContent); + # Put file + my $strCommand = + "$self->{strCommand}" . (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . + ($bRaw ? ' --raw' : '') . ' repo-put ' . $self->escape($strFile); + + my $oBuffer = new pgBackRest::Common::Io::Buffered( + new pgBackRest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax}); + my $oProcess = new pgBackRest::Common::Io::Process($oBuffer, $strCommand); + + if (defined($tContent)) + { + $oBuffer->write(\$tContent); + } + + close($oBuffer->handleWrite()); + + my $tResult; + + while (!$oBuffer->eof()) + { + $oBuffer->read(\$tResult, $self->{lBufferMax}, false); + } + + close($oBuffer->handleRead()); + $oProcess->close(); # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'lSize', value => $lSize, trace => true}, - ); + return logDebugReturn($strOperation); } #################################################################################################################################### @@ -504,20 +515,15 @@ sub remove my ( $strOperation, - $xFileExp, - $bIgnoreMissing, + $strFile, ) = logDebugParam ( __PACKAGE__ . '->remove', \@_, {name => 'xFileExp'}, - {name => 'bIgnoreMissing', optional => true, default => true}, ); - foreach my $strFileExp (ref($xFileExp) ? @{$xFileExp} : ($xFileExp)) - { - $self->{oStorageC}->remove($strFileExp, $bIgnoreMissing); - } + $self->exec("repo-rm " . $self->escape($strFile)); # Return from function and log return values if any return logDebugReturn($strOperation); @@ -527,8 +533,6 @@ sub remove # Getters #################################################################################################################################### sub capability {shift->type() eq STORAGE_POSIX} -sub type {shift->{oStorageC}->type()} -sub cipherType {shift->{strCipherType}} -sub cipherPassUser {shift->{strCipherPass}} +sub type {shift->{strType}} 1; diff --git a/lib/pgBackRest/Storage/StorageRead.pm b/lib/pgBackRest/Storage/StorageRead.pm deleted file mode 100644 index 0e493530e..000000000 --- a/lib/pgBackRest/Storage/StorageRead.pm +++ /dev/null @@ -1,143 +0,0 @@ -#################################################################################################################################### -# C Storage Read Interface -#################################################################################################################################### -package pgBackRest::Storage::StorageRead; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use File::Basename qw(dirname); -use Fcntl qw(:mode); -use File::stat qw{lstat}; -use JSON::PP; - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Storage::Base; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{oStorage}, - $self->{oStorageCRead}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oStorage'}, - {name => 'oStorageCRead'}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# Open the file -#################################################################################################################################### -sub open -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->open'); - - return logDebugReturn - ( - $strOperation, - {name => 'bResult', value => $self->{oStorageCRead}->open() ? true : false, trace => true}, - ); -} - -#################################################################################################################################### -# Read data -#################################################################################################################################### -sub read -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ( - $strOperation, - $rtBuffer, - $iSize, - ) = - logDebugParam - ( - __PACKAGE__ . '->read', \@_, - {name => 'rtBuffer'}, - {name => 'iSize'}, - ); - - # Read if not eof - my $iActualSize = 0; - - if (!$self->eof()) - { - my $tBuffer = $self->{oStorageCRead}->read($iSize); - $iActualSize = length($tBuffer); - $$rtBuffer .= $tBuffer; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iActualSize', value => $iActualSize} - ); -} - -#################################################################################################################################### -# Is the file at eof? -#################################################################################################################################### -sub eof -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->eof'); - - return logDebugReturn - ( - $strOperation, - {name => 'bResult', value => $self->{oStorageCRead}->eof() ? true : false, trace => true}, - ); -} - -#################################################################################################################################### -# Close the file -#################################################################################################################################### -sub close -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->close'); - - $self->{oStorageCRead}->close(); - - return logDebugReturn - ( - $strOperation, - {name => 'bResult', value => true, trace => true}, - ); -} - -1; diff --git a/lib/pgBackRest/Storage/StorageWrite.pm b/lib/pgBackRest/Storage/StorageWrite.pm deleted file mode 100644 index ac9190ae7..000000000 --- a/lib/pgBackRest/Storage/StorageWrite.pm +++ /dev/null @@ -1,116 +0,0 @@ -#################################################################################################################################### -# C Storage Write Interface -#################################################################################################################################### -package pgBackRest::Storage::StorageWrite; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use File::Basename qw(dirname); -use Fcntl qw(:mode); -use File::stat qw{lstat}; -use JSON::PP; - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Storage::Base; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{oStorage}, - $self->{oStorageCWrite}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oStorage'}, - {name => 'oStorageCWrite'}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# Open the file -#################################################################################################################################### -sub open -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->open'); - - $self->{oStorageCWrite}->open(); - - return logDebugReturn - ( - $strOperation, - {name => 'bResult', value => true, trace => true}, - ); -} - -#################################################################################################################################### -# Write data -#################################################################################################################################### -sub write -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ( - $strOperation, - $rtBuffer, - ) = - logDebugParam - ( - __PACKAGE__ . '->write', \@_, - {name => 'rtBuffer'}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iActualSize', value => $self->{oStorageCWrite}->write($$rtBuffer)} - ); -} - -#################################################################################################################################### -# Close the file -#################################################################################################################################### -sub close -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->close'); - - $self->{oStorageCWrite}->close(); - - return logDebugReturn - ( - $strOperation, - {name => 'bResult', value => true, trace => true}, - ); -} - -1; diff --git a/libc/LibC.h b/libc/LibC.h deleted file mode 100644 index 18b6febb0..000000000 --- a/libc/LibC.h +++ /dev/null @@ -1,78 +0,0 @@ -/*********************************************************************************************************************************** -Helper macros for LibC.xs -***********************************************************************************************************************************/ - -/*********************************************************************************************************************************** -Package Names -***********************************************************************************************************************************/ -#define PACKAGE_NAME "pgBackRest" -#define PACKAGE_NAME_LIBC PACKAGE_NAME "::LibC" - -/*********************************************************************************************************************************** -Error handling macros that throw a Perl error when a C error is caught -***********************************************************************************************************************************/ -#define ERROR_XS_BEGIN() \ - TRY_BEGIN() - -#define ERROR_XS() \ - croak("PGBRCLIB:%d:%s:%d:%s", errorCode(), errorFileName(), errorFileLine(), errorMessage()); - -#define ERROR_XS_END() \ - CATCH_ANY() \ - { \ - ERROR_XS(); \ - } \ - TRY_END(); - -/*********************************************************************************************************************************** -Simplifies switching to a temp memory context in functions and includes error handling -***********************************************************************************************************************************/ -#define MEM_CONTEXT_XS_TEMP() \ - MEM_CONTEXT_XS_TEMP_memContext - -#define MEM_CONTEXT_XS_TEMP_BEGIN() \ -{ \ - /* Create temp memory context */ \ - MemContext *MEM_CONTEXT_XS_TEMP() = memContextNew("temporary"); \ - \ - /* Switch to temp memory context */ \ - memContextSwitch(MEM_CONTEXT_XS_TEMP()); \ - \ - /* Store any errors to be croaked to Perl at the end */ \ - bool MEM_CONTEXT_XS_croak = false; \ - \ - /* Try the statement block */ \ - TRY_BEGIN() - -#define MEM_CONTEXT_XS_TEMP_END() \ - /* Set error to be croak to Perl later */ \ - CATCH_ANY() \ - { \ - MEM_CONTEXT_XS_croak = true; \ - } \ - /* Free the context on error */ \ - FINALLY() \ - { \ - memContextSwitchBack(); \ - memContextDiscard(); \ - } \ - TRY_END(); \ - \ - /* Croak on error */ \ - if (MEM_CONTEXT_XS_croak) \ - { \ - ERROR_XS() \ - } \ -} - -/*********************************************************************************************************************************** -Create new string from an SV -***********************************************************************************************************************************/ -#define STR_NEW_SV(param) \ - (SvOK(param) ? strNewN(SvPV_nolen(param), SvCUR(param)) : NULL) - -/*********************************************************************************************************************************** -Create const buffer from an SV -***********************************************************************************************************************************/ -#define BUF_CONST_SV(param) \ - (SvOK(param) ? BUF(SvPV_nolen(param), SvCUR(param)) : NULL) diff --git a/libc/LibC.xs b/libc/LibC.xs deleted file mode 100644 index f2ad85456..000000000 --- a/libc/LibC.xs +++ /dev/null @@ -1,98 +0,0 @@ -/*********************************************************************************************************************************** -C to Perl Interface - -The following C types are mapped by the current typemap: - -'AV *', 'Boolean', 'CV *', 'FILE *', 'FileHandle', 'HV *', 'I16', 'I32', 'I8', 'IV', 'InOutStream', 'InputStream', 'NV', -'OutputStream', 'PerlIO *', 'Result', 'STRLEN', 'SV *', 'SVREF', 'SysRet', 'SysRetLong', 'Time_t *', 'U16', 'U32', 'U8', 'UV', -'bool', 'bool_t', 'caddr_t', 'char', 'char *', 'char **', 'const char *', 'double', 'float', 'int', 'long', 'short', 'size_t', -'ssize_t', 'time_t', 'unsigned', 'unsigned char', 'unsigned char *', 'unsigned int', 'unsigned long', 'unsigned long *', -'unsigned short', 'void *', 'wchar_t', 'wchar_t *' -***********************************************************************************************************************************/ -#include "build.auto.h" - -#define PERL_NO_GET_CONTEXT - -/*********************************************************************************************************************************** -Perl includes - -Order is critical here so don't change it. -***********************************************************************************************************************************/ -#if __GNUC__ > 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ > 8 || (__GNUC_MINOR__ == 8 && __GNUC_PATCHLEVEL__ >= 0))) - #define WARNING_MAYBE_INITIALIZED 1 -#elif __GNUC__ > 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ > 6 || (__GNUC_MINOR__ == 6 && __GNUC_PATCHLEVEL__ >= 0))) - #define WARNING_INITIALIZED 1 -#endif - -#if WARNING_MAYBE_INITIALIZED - #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#elif WARNING_INITIALIZED - #pragma GCC diagnostic ignored "-Wuninitialized" -#endif - -#pragma GCC diagnostic ignored "-Wsign-conversion" -#pragma GCC diagnostic ignored "-Wconversion" - -#include -#include -#include - -#if WARNING_MAYBE_INITIALIZED - #pragma GCC diagnostic warning "-Wmaybe-uninitialized" -#elif WARNING_INITIALIZED - #pragma GCC diagnostic warning "-Wuninitialized" -#endif - -/*********************************************************************************************************************************** -C includes - -These includes are from the src directory. There is no Perl-specific code in them. -***********************************************************************************************************************************/ -#include "common/error.h" -#include "common/io/io.h" -#include "config/config.h" -#include "config/define.h" -#include "config/load.h" -#include "config/parse.h" -#include "storage/posix/storage.h" - -/*********************************************************************************************************************************** -Helper macros -***********************************************************************************************************************************/ -#include "LibC.h" - -/*********************************************************************************************************************************** -XSH includes - -These includes define data structures that are required for the C to Perl interface but are not part of the regular C source. -***********************************************************************************************************************************/ -#include "xs/config/configTest.xsh" -#include "xs/storage/storage.xsh" -#include "xs/storage/storageRead.xsh" -#include "xs/storage/storageWrite.xsh" - -/*********************************************************************************************************************************** -Module definition -***********************************************************************************************************************************/ -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC -PROTOTYPES: DISABLE - -# Return UVSIZE to ensure that this Perl supports 64-bit integers -# ---------------------------------------------------------------------------------------------------------------------------------- -I32 -libcUvSize() -CODE: - RETVAL = UVSIZE; -OUTPUT: - RETVAL - -# Exported functions and modules -# -# These modules should map 1-1 with C modules in src directory. -# ---------------------------------------------------------------------------------------------------------------------------------- -INCLUDE: xs/config/config.xs -INCLUDE: xs/config/configTest.xs -INCLUDE: xs/config/define.xs -INCLUDE: xs/storage/storage.xs -INCLUDE: xs/storage/storageRead.xs -INCLUDE: xs/storage/storageWrite.xs diff --git a/libc/Makefile.PL b/libc/Makefile.PL deleted file mode 100644 index 60cd546fc..000000000 --- a/libc/Makefile.PL +++ /dev/null @@ -1,147 +0,0 @@ -#################################################################################################################################### -# Build Makefile and Auto-Generate Files Required for Build -# -# The C library is only used for Perl unit tests. For a production build all C library exports are built directly into the -# pgbackrest executable. See src/perl/libc.auto.c. -#################################################################################################################################### -use 5.010001; -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -# Convert die to confess to capture the stack trace -$SIG{__DIE__} = sub { Carp::confess @_ }; - -use Cwd qw(abs_path); -use ExtUtils::MakeMaker; -use File::Basename qw(dirname); - -#################################################################################################################################### -# Storage object to use for all file operations -#################################################################################################################################### -use constant PROJECT_NAME => 'pgBackRest'; - -#################################################################################################################################### -# Make sure the makefile is being created in an expected test directory. This should prevent users from building it in production. -#################################################################################################################################### -if (dirname($0) !~ /\.vagrant\/bin\/[^\/]+\/libc$/) -{ - confess - "LibC is not being built in a test directory (" . dirname($0) . - "). LibC should not be distributed for production builds. See build documentation for details."; -} - -#################################################################################################################################### -# Create C Makefile -#################################################################################################################################### -my $strBuildPath = dirname(dirname(abs_path($0))); - -# Create C files array -my @stryCFile = -( - 'LibC.c', - - 'command/command.c', - 'common/compress/gz/common.c', - 'common/compress/gz/compress.c', - 'common/compress/gz/decompress.c', - 'common/compress/helper.c', - 'common/crypto/cipherBlock.c', - 'common/crypto/common.c', - 'common/crypto/hash.c', - 'common/debug.c', - 'common/encode.c', - 'common/encode/base64.c', - 'common/error.c', - 'common/ini.c', - 'common/io/bufferRead.c', - 'common/io/bufferWrite.c', - 'common/io/filter/buffer.c', - 'common/io/filter/filter.c', - 'common/io/filter/group.c', - 'common/io/filter/sink.c', - 'common/io/filter/size.c', - 'common/io/handleWrite.c', - 'common/io/http/cache.c', - 'common/io/http/client.c', - 'common/io/http/common.c', - 'common/io/http/header.c', - 'common/io/http/query.c', - 'common/io/io.c', - 'common/io/read.c', - 'common/io/tls/client.c', - 'common/io/write.c', - 'common/log.c', - 'common/memContext.c', - 'common/regExp.c', - 'common/stackTrace.c', - 'common/time.c', - 'common/type/convert.c', - 'common/type/buffer.c', - 'common/type/json.c', - 'common/type/keyValue.c', - 'common/type/list.c', - 'common/type/string.c', - 'common/type/stringList.c', - 'common/type/variant.c', - 'common/type/variantList.c', - 'common/type/xml.c', - 'common/user.c', - 'common/wait.c', - 'config/config.c', - 'config/define.c', - 'config/load.c', - 'config/parse.c', - 'protocol/client.c', - 'protocol/command.c', - 'protocol/helper.c', - 'protocol/parallel.c', - 'protocol/parallelJob.c', - 'protocol/server.c', - 'storage/posix/read.c', - 'storage/posix/storage.c', - 'storage/posix/write.c', - 'storage/s3/read.c', - 'storage/s3/storage.c', - 'storage/s3/write.c', - 'storage/helper.c', - 'storage/read.c', - 'storage/storage.c', - 'storage/write.c', -); - -# Add ../src for files that are outside libc -for (my $iFileIdx = 1; $iFileIdx < @stryCFile; $iFileIdx++) -{ - $stryCFile[$iFileIdx] = '../src/' . $stryCFile[$iFileIdx]; -} - -# Write the makefile -WriteMakefile -( - NAME => PROJECT_NAME . '::LibC', - VERSION => '999', - AUTHOR => 'David Steele ', - - CCFLAGS => join(' ', qw) - -Wfatal-errors -Wall -Wextra -Wwrite-strings -Wno-clobbered -Wno-missing-field-initializers - -o $@ - -std=c99 - -D_POSIX_C_SOURCE=200112L - -D_FILE_OFFSET_BITS=64 - `pkg-config libxml-2.0 --cflags` - -I`pg_config --includedir` - )), - - INC => join(' ', qw( - -I. - -I../src - )), - - C => \@stryCFile, - - LIBS => '-lcrypto -lpq -lssl -lxml2', - - OBJECT => '$(O_FILES)', -); diff --git a/libc/build/lib/pgBackRestLibC/Build.pm b/libc/build/lib/pgBackRestLibC/Build.pm deleted file mode 100644 index edd6be115..000000000 --- a/libc/build/lib/pgBackRestLibC/Build.pm +++ /dev/null @@ -1,372 +0,0 @@ -#################################################################################################################################### -# Auto-Generate XS and PM Files Required for Perl -#################################################################################################################################### -package pgBackRestLibC::Build; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use lib dirname($0) . '/../build/lib'; -use lib dirname($0) . '/../lib'; - -use pgBackRest::Common::Log; -use pgBackRest::Common::String; -use pgBackRest::Version; - -use pgBackRestBuild::Build; -use pgBackRestBuild::Build::Common; -use pgBackRestBuild::Config::Data; -use pgBackRestBuild::Error::Data; - -use pgBackRestTest::Common::Storage; -use pgBackRestTest::Common::StoragePosix; - -#################################################################################################################################### -# Perl function and constant exports -#################################################################################################################################### -use constant BLD_EXPORTTYPE_SUB => 'sub'; -use constant BLD_EXPORTTYPE_CONSTANT => 'constant'; - -use constant LIB_AUTO_NAME => 'LibCAuto'; - -#################################################################################################################################### -# Save contents to a file if the file is missing or the contents are different. This saves write IO and prevents the timestamp from -# changing. -#################################################################################################################################### -sub buildPutDiffers -{ - my $oStorage = shift; - my $strFile = shift; - my $strContents = shift; - - # Attempt to load the file - my $bSave = true; - my $oFile = $oStorage->openRead($strFile, {bIgnoreMissing => true}); - - # If file was found see if the content is the same - if (defined($oFile) && ${$oStorage->get($oFile)} eq $strContents) - { - $bSave = false; - } - - # Save if the contents are different or missing - if ($bSave) - { - $oStorage->put($strFile, $strContents); - } - - return $bSave; -} - -#################################################################################################################################### -# Static exports -#################################################################################################################################### -my $rhExport = -{ - 'config' => - { - &BLD_EXPORTTYPE_SUB => [qw( - cfgCommandName - cfgOptionIndexTotal - cfgOptionName - )], - }, - - 'configDefine' => - { - &BLD_EXPORTTYPE_SUB => [qw( - cfgCommandId - cfgDefOptionDefault - cfgDefOptionPrefix - cfgDefOptionSecure - cfgDefOptionType - cfgDefOptionValid - cfgOptionId - cfgOptionTotal - )], - }, - - 'debug' => - { - &BLD_EXPORTTYPE_SUB => [qw( - libcUvSize - )], - }, - - 'storage' => - { - &BLD_EXPORTTYPE_SUB => [qw( - storageRepoFree - )], - }, - - 'test' => - { - &BLD_EXPORTTYPE_SUB => [qw( - cfgParseTest - )], - }, -}; - -#################################################################################################################################### -# Execute all build functions -#################################################################################################################################### -sub buildXsAll -{ - my $strBuildPath = shift; - - # List of files built - my @stryBuilt; - - # Storage - my $oStorage = new pgBackRestTest::Common::Storage( - $strBuildPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); - - # Build interface file - my $strContent = - ('#' x 132) . "\n" . - '# ' . bldAutoWarning('Build.pm') . "\n" . - ('#' x 132) . "\n" . - "package pgBackRest::LibCAuto;\n" . - "\n" . - "use strict;\n" . - "use warnings;\n"; - - # Generate constants for options that have a list of strings as allowed values - my $rhConfigDefine = cfgDefine(); - my $strConstantBlock; - - foreach my $strOption (sort(keys(%{$rhConfigDefine}))) - { - my $rhOption = $rhConfigDefine->{$strOption}; - - next if $rhOption->{&CFGDEF_TYPE} ne CFGDEF_TYPE_STRING; - next if $strOption =~ /^log-level-/; - - if (defined($rhOption->{&CFGDEF_ALLOW_LIST})) - { - $strConstantBlock .= defined($strConstantBlock) ? "\n" : ''; - - foreach my $strValue (@{$rhOption->{&CFGDEF_ALLOW_LIST}}) - { - my $strConstant = 'CFGOPTVAL_' . uc("${strOption}_${strValue}"); - $strConstant =~ s/\-/\_/g; - - $strConstantBlock .= " ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> '${strValue}',\n"; - push(@{$rhExport->{'config'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant); - } - } - - foreach my $strCommand (sort(keys(%{$rhOption->{&CFGDEF_COMMAND}}))) - { - my $rhCommand = $rhOption->{&CFGDEF_COMMAND}{$strCommand}; - - if (defined($rhCommand->{&CFGDEF_ALLOW_LIST})) - { - $strConstantBlock .= defined($strConstantBlock) ? "\n" : ''; - - foreach my $strValue (@{$rhCommand->{&CFGDEF_ALLOW_LIST}}) - { - my $strConstant = 'CFGOPTVAL_' . uc("${strCommand}_${strOption}_${strValue}"); - $strConstant =~ s/\-/\_/g; - - $strConstantBlock .= " ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> '${strValue}',\n"; - push(@{$rhExport->{'config'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant); - } - } - } - } - - # Generate command constants - foreach my $strCommand (cfgDefineCommandList()) - { - my $strConstant = "CFGCMD_" . uc($strCommand); - $strConstant =~ s/\-/\_/g; - - push(@{$rhExport->{'config'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant); - } - - # Generate option constants - foreach my $strOption (sort(keys(%{$rhConfigDefine}))) - { - # Build Perl constant - my $strConstant = "CFGOPT_" . uc($strOption); - $strConstant =~ s/\-/\_/g; - - # Builds option data - for (my $iOptionIndex = 1; $iOptionIndex <= $rhConfigDefine->{$strOption}{&CFGDEF_INDEX_TOTAL}; $iOptionIndex++) - { - push(@{$rhExport->{'config'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant . ($iOptionIndex == 1 ? '' : $iOptionIndex)); - } - } - - # Generate option type constants - $strConstantBlock .= defined($strConstantBlock) ? "\n" : ''; - my $iIndex = 0; - - foreach my $strOptionType (cfgDefineOptionTypeList()) - { - # Build Perl constant - my $strConstant = "CFGDEF_TYPE_" . uc($strOptionType); - $strConstant =~ s/\-/\_/g; - - $strConstantBlock .= - " ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> $iIndex,\n"; - push(@{$rhExport->{'configDefine'}{&BLD_EXPORTTYPE_CONSTANT}}, $strConstant); - - $iIndex++; - }; - - # Generate encode type constants - $strConstantBlock .= defined($strConstantBlock) ? "\n" : ''; - - my $strConstant = "ENCODE_TYPE_BASE64"; - $strConstantBlock .= - " ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> 0,\n"; - - # Generate cipher type constants - $strConstantBlock .= defined($strConstantBlock) ? "\n" : ''; - - $strConstant = "CIPHER_MODE_ENCRYPT"; - $strConstantBlock .= - " ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> 0,\n"; - $strConstant = "CIPHER_MODE_DECRYPT"; - $strConstantBlock .= - " ${strConstant}" . (' ' x (69 - length($strConstant) - 4)) . "=> 1,\n"; - - $strContent .= - "\n" . - "# Configuration option value constants\n" . - "sub libcAutoConstant\n" . - "{\n" . - " return\n" . - " {\n" . - " " . trim($strConstantBlock) . "\n" . - " }\n" . - "}\n"; - - # Generate export tags - my $strExportTags; - - foreach my $strSection (sort(keys(%{$rhExport}))) - { - my $rhExportSection = $rhExport->{$strSection}; - - $strExportTags .= (defined($strExportTags) ? "\n" : '') . " ${strSection} =>\n [\n"; - - if (defined($rhExportSection->{&BLD_EXPORTTYPE_CONSTANT}) && @{$rhExportSection->{&BLD_EXPORTTYPE_CONSTANT}} > 0) - { - foreach my $strConstant (@{$rhExportSection->{&BLD_EXPORTTYPE_CONSTANT}}) - { - $strExportTags .= " '${strConstant}',\n"; - } - } - - if (defined($rhExportSection->{&BLD_EXPORTTYPE_SUB}) && @{$rhExportSection->{&BLD_EXPORTTYPE_SUB}} > 0) - { - foreach my $strSub (@{$rhExportSection->{&BLD_EXPORTTYPE_SUB}}) - { - $strExportTags .= " '${strSub}',\n"; - } - } - - $strExportTags .= " ],\n"; - } - - $strContent .= - "\n" . - "# Export function and constants\n" . - "sub libcAutoExportTag\n" . - "{\n" . - " return\n" . - " {\n" . - $strExportTags . - " }\n" . - "}\n"; - - # Add module end marker - $strContent .= - "\n" . - "1;\n"; - - # Save the file - my $strFile = 'lib/' . PROJECT_NAME . '/' . LIB_AUTO_NAME . '.pm'; - - if (buildPutDiffers($oStorage, "../${strFile}", $strContent)) - { - push(@stryBuilt, $strFile); - } - - # Build error file - #------------------------------------------------------------------------------------------------------------------------------- - my $rhErrorDefine = errorDefine(); - - # Order by id for the list that is id ordered - my $rhErrorId = {}; - - foreach my $strType (sort(keys(%{$rhErrorDefine}))) - { - my $iCode = $rhErrorDefine->{$strType}; - $rhErrorId->{$iCode} = $strType; - } - - # Output errors - $strContent = - ('#' x 132) . "\n" . - "# COMMON EXCEPTION AUTO MODULE\n" . - "# \n" . - '# ' . bldAutoWarning('Build.pm') . "\n" . - ('#' x 132) . "\n" . - "package pgBackRest::Common::ExceptionAuto;\n" . - "\n" . - "use strict;\n" . - "use warnings FATAL => qw(all);\n" . - "\n" . - "use Exporter qw(import);\n" . - " our \@EXPORT = qw();\n" . - "\n" . - ('#' x 132) . "\n" . - "# Error Definitions\n" . - ('#' x 132) . "\n" . - "use constant ERROR_MINIMUM => " . ERRDEF_MIN . ";\n" . - "push \@EXPORT, qw(ERROR_MINIMUM);\n" . - "use constant ERROR_MAXIMUM => " . ERRDEF_MAX . ";\n" . - "push \@EXPORT, qw(ERROR_MAXIMUM);\n" . - "\n"; - - foreach my $iCode (sort({sprintf("%03d", $a) cmp sprintf("%03d", $b)} keys(%{$rhErrorId}))) - { - my $strType = "ERROR_" . uc($rhErrorId->{$iCode}); - $strType =~ s/\-/\_/g; - - $strContent .= - "use constant ${strType}" . (' ' x (54 - length($strType))) . " => $iCode;\n" . - "push \@EXPORT, qw(${strType});\n" - } - - $strContent .= - "\n" . - "1;\n"; - - $strFile = 'lib/' . PROJECT_NAME . '/Common/ExceptionAuto.pm'; - - if (buildPutDiffers($oStorage, "../${strFile}", $strContent)) - { - push(@stryBuilt, $strFile); - } - - # Return list of files built - return @stryBuilt; -} - -push @EXPORT, qw(buildXsAll); - -1; diff --git a/libc/typemap b/libc/typemap deleted file mode 100644 index 14e50c447..000000000 --- a/libc/typemap +++ /dev/null @@ -1,3 +0,0 @@ -pgBackRest::LibC::Storage T_PTROBJ -pgBackRest::LibC::StorageRead T_PTROBJ -pgBackRest::LibC::StorageWrite T_PTROBJ diff --git a/libc/xs/config/config.xs b/libc/xs/config/config.xs deleted file mode 100644 index 18bf17087..000000000 --- a/libc/xs/config/config.xs +++ /dev/null @@ -1,17 +0,0 @@ -# ---------------------------------------------------------------------------------------------------------------------------------- -# Config Perl Exports -# ---------------------------------------------------------------------------------------------------------------------------------- - -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC - -const char * -cfgCommandName(commandId) - U32 commandId - -I32 -cfgOptionIndexTotal(optionId) - U32 optionId - -const char * -cfgOptionName(optionId) - U32 optionId diff --git a/libc/xs/config/configTest.xs b/libc/xs/config/configTest.xs deleted file mode 100644 index f47c958f2..000000000 --- a/libc/xs/config/configTest.xs +++ /dev/null @@ -1,38 +0,0 @@ -# ---------------------------------------------------------------------------------------------------------------------------------- -# Config Test Perl Exports -# ---------------------------------------------------------------------------------------------------------------------------------- - -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC - -#################################################################################################################################### -# Parse command line and return a JSON object with results -#################################################################################################################################### -SV * -cfgParseTest(backrestBin, parseParam) - const char *backrestBin - const char *parseParam -CODE: - RETVAL = NULL; - - ERROR_XS_BEGIN() - { - // This should run in a temp context but for some reason getopt_long gets upset when if gets called again after the previous - // arg list being freed. So, this is a memory leak but it is only used for testing, not production. - StringList *paramList = strLstNewSplitZ(strCat(strNew("pgbackrest|"), parseParam), "|"); - - // Don't use cfgLoad() because it has a lot of side effects that we don't want - configParse(strLstSize(paramList), strLstPtr(paramList), false); - cfgExeSet(strNew(backrestBin)); - cfgLoadUpdateOption(); - - String *result = perlOptionJson(); - - RETVAL = newSV(strSize(result)); - SvPOK_only(RETVAL); - - strcpy(SvPV_nolen(RETVAL), strPtr(result)); - SvCUR_set(RETVAL, strSize(result)); - } - ERROR_XS_END() -OUTPUT: - RETVAL diff --git a/libc/xs/config/configTest.xsh b/libc/xs/config/configTest.xsh deleted file mode 100644 index 1a36ee18d..000000000 --- a/libc/xs/config/configTest.xsh +++ /dev/null @@ -1,122 +0,0 @@ -/*********************************************************************************************************************************** -Config XS Header -***********************************************************************************************************************************/ -#include - -#include "common/memContext.h" -#include "common/type/json.h" -#include "config/config.h" - -/*********************************************************************************************************************************** -Build JSON output from options -***********************************************************************************************************************************/ -String * -perlOptionJson(void) -{ - String *result = NULL; - - MEM_CONTEXT_TEMP_BEGIN() - { - KeyValue *configKv = kvNew(); - - for (ConfigOption optionId = 0; optionId < CFG_OPTION_TOTAL; optionId++) - { - // Skip if not valid - if (!cfgOptionValid(optionId)) - continue; - - Variant *optionVar = varNewKv(kvNew()); - - // Add valid - kvPut(varKv(optionVar), VARSTRDEF("valid"), BOOL_TRUE_VAR); - - // Add source - const Variant *source = NULL; - - switch (cfgOptionSource(optionId)) - { - case cfgSourceParam: - { - source = varNewStrZ("param"); - break; - } - - case cfgSourceConfig: - { - source = varNewStrZ("config"); - break; - } - - case cfgSourceDefault: - { - source = varNewStrZ("default"); - break; - } - } - - kvPut(varKv(optionVar), VARSTRDEF("source"), source); - - // Add negate and reset - kvPut(varKv(optionVar), VARSTRDEF("negate"), VARBOOL(cfgOptionNegate(optionId))); - kvPut(varKv(optionVar), VARSTRDEF("reset"), VARBOOL(cfgOptionReset(optionId))); - - // Add value if it is set - if (cfgOptionTest(optionId)) - { - const Variant *valueVar = NULL; - - switch (cfgDefOptionType(cfgOptionDefIdFromId(optionId))) - { - case cfgDefOptTypeBoolean: - case cfgDefOptTypeFloat: - case cfgDefOptTypeInteger: - case cfgDefOptTypePath: - case cfgDefOptTypeSize: - case cfgDefOptTypeString: - { - valueVar = cfgOption(optionId); - break; - } - - case cfgDefOptTypeHash: - { - valueVar = varNewKv(kvNew()); - - const KeyValue *valueKv = cfgOptionKv(optionId); - const VariantList *keyList = kvKeyList(valueKv); - - for (unsigned int listIdx = 0; listIdx < varLstSize(keyList); listIdx++) - kvPut(varKv(valueVar), varLstGet(keyList, listIdx), kvGet(valueKv, varLstGet(keyList, listIdx))); - - break; - } - - case cfgDefOptTypeList: - { - valueVar = varNewKv(kvNew()); - - const VariantList *valueList = cfgOptionLst(optionId); - - for (unsigned int listIdx = 0; listIdx < varLstSize(valueList); listIdx++) - kvPut(varKv(valueVar), varLstGet(valueList, listIdx), BOOL_TRUE_VAR); - - break; - } - } - - kvPut(varKv(optionVar), VARSTRDEF("value"), valueVar); - } - - kvPut(configKv, VARSTRZ(cfgOptionName(optionId)), optionVar); - } - - MEM_CONTEXT_PRIOR_BEGIN() - { - result = jsonFromKv(configKv); - } - MEM_CONTEXT_PRIOR_END(); - } - MEM_CONTEXT_TEMP_END(); - - return result; -} diff --git a/libc/xs/config/define.xs b/libc/xs/config/define.xs deleted file mode 100644 index f73d05b30..000000000 --- a/libc/xs/config/define.xs +++ /dev/null @@ -1,112 +0,0 @@ -# ---------------------------------------------------------------------------------------------------------------------------------- -# Config Definition Perl Exports -# ---------------------------------------------------------------------------------------------------------------------------------- - -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC - -I32 -cfgCommandId(commandName) - const char *commandName -CODE: - RETVAL = 0; - - ERROR_XS_BEGIN() - { - RETVAL = cfgCommandId(commandName, true); - } - ERROR_XS_END(); -OUTPUT: - RETVAL - -I32 -cfgOptionId(optionName) - const char *optionName -CODE: - RETVAL = 0; - - ERROR_XS_BEGIN() - { - RETVAL = cfgOptionId(optionName); - } - ERROR_XS_END(); -OUTPUT: - RETVAL - -const char * -cfgDefOptionDefault(commandId, optionId) - U32 commandId - U32 optionId -CODE: - RETVAL = NULL; - - ERROR_XS_BEGIN() - { - RETVAL = cfgDefOptionDefault(cfgCommandDefIdFromId(commandId), cfgOptionDefIdFromId(optionId)); - } - ERROR_XS_END(); -OUTPUT: - RETVAL - -const char * -cfgDefOptionPrefix(optionId) - U32 optionId -CODE: - RETVAL = NULL; - - ERROR_XS_BEGIN() - { - RETVAL = cfgDefOptionPrefix(cfgOptionDefIdFromId(optionId)); - } - ERROR_XS_END(); -OUTPUT: - RETVAL - -bool -cfgDefOptionSecure(optionId) - U32 optionId -CODE: - RETVAL = false; - - ERROR_XS_BEGIN() - { - RETVAL = cfgDefOptionSecure(cfgOptionDefIdFromId(optionId)); - } - ERROR_XS_END(); -OUTPUT: - RETVAL - -I32 -cfgDefOptionType(optionId); - U32 optionId -CODE: - RETVAL = 0; - - ERROR_XS_BEGIN() - { - RETVAL = cfgDefOptionType(cfgOptionDefIdFromId(optionId)); - } - ERROR_XS_END(); -OUTPUT: - RETVAL - -bool -cfgDefOptionValid(commandId, optionId) - U32 commandId - U32 optionId -CODE: - RETVAL = false; - - ERROR_XS_BEGIN() - { - RETVAL = cfgDefOptionValid(cfgCommandDefIdFromId(commandId), cfgOptionDefIdFromId(optionId)); - } - ERROR_XS_END(); -OUTPUT: - RETVAL - -U32 -cfgOptionTotal() -CODE: - RETVAL = CFG_OPTION_TOTAL; -OUTPUT: - RETVAL diff --git a/libc/xs/storage/storage.xs b/libc/xs/storage/storage.xs deleted file mode 100644 index 1104d64f6..000000000 --- a/libc/xs/storage/storage.xs +++ /dev/null @@ -1,356 +0,0 @@ -# ---------------------------------------------------------------------------------------------------------------------------------- -# Storage Exports -# ---------------------------------------------------------------------------------------------------------------------------------- - -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::Storage - -#################################################################################################################################### -pgBackRest::LibC::Storage -new(class, type, path) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - const String *class = STR_NEW_SV($arg); - const String *type = STR_NEW_SV($arg); - const String *path = STR_NEW_SV($arg); -CODE: - CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::Storage")); - - if (strEqZ(type, "")) - { - MEM_CONTEXT_PRIOR_BEGIN() - { - RETVAL = storagePosixNew( - path == NULL ? STRDEF("/") : path, STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); - } - MEM_CONTEXT_PRIOR_END(); - } - else if (strEqZ(type, "")) - { - CHECK(path == NULL); - RETVAL = (Storage *)storageRepoWrite(); - } - else if (strEqZ(type, "")) - { - CHECK(path == NULL); - - MEM_CONTEXT_PRIOR_BEGIN() - { - RETVAL = storagePosixNew(cfgOptionStr(cfgOptPgPath), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); - } - MEM_CONTEXT_PRIOR_END(); - } - else - THROW_FMT(AssertError, "unexpected storage type '%s'", strPtr(type)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -bucketCreate(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::Storage self -CODE: - if (strEq(storageType(self), STORAGE_S3_TYPE_STR)) - storageS3Request((StorageS3 *)storageDriver(self), HTTP_VERB_PUT_STR, FSLASH_STR, NULL, NULL, true, false); - else - THROW_FMT(AssertError, "unable to create bucket on '%s' storage", strPtr(storageType(self))); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -bool -copy(self, source, destination) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead source - pgBackRest::LibC::StorageWrite destination -CODE: - RETVAL = storageCopyP(source, destination); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -SV * -get(self, read) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead read -CODE: - RETVAL = NULL; - Buffer *buffer = storageGetP(read); - - if (buffer != NULL) - { - if (bufUsed(buffer) == 0) - RETVAL = newSVpv("", 0); - else - RETVAL = newSVpv((char *)bufPtr(buffer), bufUsed(buffer)); - } -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -SV * -info(self, pathExp, ignoreMissing) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::Storage self - const String *pathExp = STR_NEW_SV($arg); - bool ignoreMissing -CODE: - RETVAL = NULL; - - StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = ignoreMissing); - - if (info.exists) - { - String *json = storageManifestXsInfo(NULL, &info); - RETVAL = newSVpv((char *)strPtr(json), strSize(json)); - } -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -SV * -list(self, pathExp, ignoreMissing, sortAsc, expression) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::Storage self - const String *pathExp = STR_NEW_SV($arg); - bool ignoreMissing - bool sortAsc - const String *expression = STR_NEW_SV($arg); -CODE: - StringList *fileList = strLstSort( - storageListP(self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false, - .expression = expression), sortAsc ? sortOrderAsc : sortOrderDesc); - - const String *fileListJson = jsonFromVar(varNewVarLst(varLstNewStrLst(fileList))); - - RETVAL = newSVpv(strPtr(fileListJson), strSize(fileListJson)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -SV * -manifest(self, pathExp, filter=NULL) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::Storage self - const String *pathExp = STR_NEW_SV($arg); - const String *filter = STR_NEW_SV($arg); -CODE: - StorageManifestXsCallbackData data = {.storage = self, .json = strNew("{"), .pathRoot = pathExp, .filter = filter}; - - // If a path is specified - StorageInfo info = storageInfoP(self, pathExp, .ignoreMissing = true); - - if (!info.exists || info.type == storageTypePath) - { - storageInfoListP( - self, data.pathRoot, storageManifestXsCallback, &data, - .errorOnMissing = storageFeature(self, storageFeaturePath) ? true : false); - } - // Else a file is specified - else - { - info.name = strBase(storagePathP(self, pathExp)); - strCat(data.json, strPtr(storageManifestXsInfo(NULL, &info))); - } - - strCat(data.json, "}"); - - RETVAL = newSVpv((char *)strPtr(data.json), strSize(data.json)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -SV * -pathGet(self, pathExp) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::Storage self - const String *pathExp = STR_NEW_SV($arg); -CODE: - String *path = storagePathP(self, pathExp); - RETVAL = newSVpv((char *)strPtr(path), strSize(path)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -pathRemove(self, pathExp, ignoreMissing, recurse) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::Storage self - const String *pathExp = STR_NEW_SV($arg); - bool ignoreMissing - bool recurse -CODE: - storagePathRemoveP( - self, pathExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false, .recurse = recurse); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -UV -put(self, write, buffer) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageWrite write - const Buffer *buffer = BUF_CONST_SV($arg); -CODE: - storagePutP(write, buffer); - RETVAL = buffer ? bufUsed(buffer) : 0; -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -bool -readDrain(self, read) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead read -CODE: - RETVAL = false; - - // Read and discard all IO (this is useful for processing filters) - if (ioReadOpen(storageReadIo(read))) - { - Buffer *buffer = bufNew(ioBufferSize()); - - do - { - ioRead(storageReadIo(read), buffer); - bufUsedZero(buffer); - } - while (!ioReadEof(storageReadIo(read))); - - ioReadClose(storageReadIo(read)); - RETVAL = true; - } -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -remove(self, fileExp, ignoreMissing) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::Storage self - const String *fileExp = STR_NEW_SV($arg); - bool ignoreMissing -CODE: - storageRemoveP(self, fileExp, .errorOnMissing = storageFeature(self, storageFeaturePath) ? !ignoreMissing : false); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -const char * -cipherType(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -CODE: - if (cfgOptionStr(cfgOptRepoCipherType) == NULL || cipherType(cfgOptionStr(cfgOptRepoCipherType)) == cipherTypeNone) - RETVAL = NULL; - else - RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherType)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -const char * -cipherPass(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -CODE: - RETVAL = strPtr(cfgOptionStr(cfgOptRepoCipherPass)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -const char * -type(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::Storage self -CODE: - RETVAL = strPtr(storageType(self)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC - -#################################################################################################################################### -void -storageRepoFree() -CODE: - storageHelperFree(); diff --git a/libc/xs/storage/storage.xsh b/libc/xs/storage/storage.xsh deleted file mode 100644 index d752aa09a..000000000 --- a/libc/xs/storage/storage.xsh +++ /dev/null @@ -1,129 +0,0 @@ -/*********************************************************************************************************************************** -Storage XS Header -***********************************************************************************************************************************/ -#include "common/assert.h" -#include "common/compress/gz/compress.h" -#include "common/compress/gz/decompress.h" -#include "common/crypto/cipherBlock.h" -#include "common/io/filter/size.h" -#include "common/memContext.h" -#include "common/type/convert.h" -#include "common/type/json.h" -#include "postgres/interface.h" -#include "storage/helper.h" -#include "storage/s3/storage.intern.h" -#include "storage/storage.intern.h" - -typedef Storage *pgBackRest__LibC__Storage; - -/*********************************************************************************************************************************** -Manifest callback -***********************************************************************************************************************************/ -typedef struct StorageManifestXsCallbackData -{ - const Storage *storage; - const String *pathRoot; - const String *path; - String *json; - const String *filter; -} StorageManifestXsCallbackData; - -String * -storageManifestXsInfo(const String *path, const StorageInfo *info) -{ - String *json = strNew(""); - - if (info->name != NULL) - { - strCatFmt( - json, "%s:", strPtr(jsonFromStr(path == NULL ? info->name : strNewFmt("%s/%s", strPtr(path), strPtr(info->name))))); - } - - strCatFmt(json, "{\"group\":%s,\"user\":%s,\"type\":\"", strPtr(jsonFromStr(info->group)), strPtr(jsonFromStr(info->user))); - - switch (info->type) - { - case storageTypeFile: - { - strCatFmt( - json, "f\",\"mode\":\"%04o\",\"modification_time\":%" PRId64 ",\"size\":%" PRIu64 "}", info->mode, - (int64_t)info->timeModified, info->size); - break; - } - - case storageTypeLink: - { - strCatFmt(json, "l\",\"link_destination\":%s}", strPtr(jsonFromStr(info->linkDestination))); - break; - } - - case storageTypePath: - { - strCatFmt(json, "d\",\"mode\":\"%04o\"}", info->mode); - break; - } - - case storageTypeSpecial: - { - strCatFmt(json, "s\",\"mode\":\"%04o\"}", info->mode); - break; - } - } - - return json; -} - -void -storageManifestXsCallback(void *callbackData, const StorageInfo *info) -{ - StorageManifestXsCallbackData *data = (StorageManifestXsCallbackData *)callbackData; - - if (data->path == NULL || !strEqZ(info->name, ".")) - { - if (!strEqZ(info->name, ".") && data->filter && !strEq(data->filter, info->name)) - return; - - if (strSize(data->json) != 1) - strCat(data->json, ","); - - strCat(data->json, strPtr(storageManifestXsInfo(data->path, info))); - - if (info->type == storageTypePath) - { - if (!strEqZ(info->name, ".")) - { - StorageManifestXsCallbackData dataSub = - { - .storage = data->storage, - .json = data->json, - .pathRoot = data->pathRoot, - .path = data->path == NULL ? info->name : strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)), - }; - - storageInfoListP( - dataSub.storage, strNewFmt("%s/%s", strPtr(dataSub.pathRoot), strPtr(dataSub.path)), storageManifestXsCallback, - &dataSub); - } - } - } -} - -/*********************************************************************************************************************************** -Add IO filter -***********************************************************************************************************************************/ -void -storageFilterXsAdd(IoFilterGroup *filterGroup, const String *filter, const String *paramJson) -{ - VariantList *paramList = paramJson ? jsonToVarLst(paramJson) : NULL; - - if (strEqZ(filter, "pgBackRest::Storage::Filter::CipherBlock")) - { - ioFilterGroupAdd( - filterGroup, - cipherBlockNew( - varUInt64Force(varLstGet(paramList, 0)) ? cipherModeEncrypt : cipherModeDecrypt, - cipherType(varStr(varLstGet(paramList, 1))), BUFSTR(varStr(varLstGet(paramList, 2))), NULL)); - } - else - THROW_FMT(AssertError, "unable to add invalid filter '%s'", strPtr(filter)); -} diff --git a/libc/xs/storage/storageRead.xs b/libc/xs/storage/storageRead.xs deleted file mode 100644 index 7d22fd461..000000000 --- a/libc/xs/storage/storageRead.xs +++ /dev/null @@ -1,126 +0,0 @@ -# ---------------------------------------------------------------------------------------------------------------------------------- -# Storage Read Exports -# ---------------------------------------------------------------------------------------------------------------------------------- - -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::StorageRead - -#################################################################################################################################### -pgBackRest::LibC::StorageRead -new(class, storage, file, ignoreMissing) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - const String *class = STR_NEW_SV($arg); - pgBackRest::LibC::Storage storage - const String *file = STR_NEW_SV($arg); - bool ignoreMissing -CODE: - CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageRead")); - - RETVAL = storageReadMove(storageNewReadP(storage, file, .ignoreMissing = ignoreMissing), memContextPrior()); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -filterAdd(self, filter, param) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead self - const String *filter = STR_NEW_SV($arg); - const String *param = STR_NEW_SV($arg); -CODE: - IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(self)); - storageFilterXsAdd(filterGroup, filter, param); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -bool -open(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead self -CODE: - RETVAL = ioReadOpen(storageReadIo(self)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -SV * -read(self, bufferSize) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead self - U32 bufferSize -CODE: - RETVAL = NEWSV(0, bufferSize); - SvPOK_only(RETVAL); - - Buffer *bufferRead = bufNewUseC((unsigned char *)SvPV_nolen(RETVAL), bufferSize); - ioRead(storageReadIo(self), bufferRead); - - SvCUR_set(RETVAL, bufUsed(bufferRead)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -bool -eof(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead self -CODE: - RETVAL = ioReadEof(storageReadIo(self)); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -close(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead self -CODE: - ioReadClose(storageReadIo(self)); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -DESTROY(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageRead self -CODE: - storageReadFree(self); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); diff --git a/libc/xs/storage/storageRead.xsh b/libc/xs/storage/storageRead.xsh deleted file mode 100644 index c2112cbc9..000000000 --- a/libc/xs/storage/storageRead.xsh +++ /dev/null @@ -1,4 +0,0 @@ -/*********************************************************************************************************************************** -Storage Read XS Header -***********************************************************************************************************************************/ -typedef StorageRead *pgBackRest__LibC__StorageRead; diff --git a/libc/xs/storage/storageWrite.xs b/libc/xs/storage/storageWrite.xs deleted file mode 100644 index 72061829e..000000000 --- a/libc/xs/storage/storageWrite.xs +++ /dev/null @@ -1,113 +0,0 @@ -# ---------------------------------------------------------------------------------------------------------------------------------- -# Storage Write Exports -# ---------------------------------------------------------------------------------------------------------------------------------- - -MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC::StorageWrite - -#################################################################################################################################### -pgBackRest::LibC::StorageWrite -new(class, storage, file, mode, user, group, timeModified, atomic, pathCreate) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - const String *class = STR_NEW_SV($arg); - pgBackRest::LibC::Storage storage - const String *file = STR_NEW_SV($arg); - U32 mode - const String *user = STR_NEW_SV($arg); - const String *group = STR_NEW_SV($arg); - IV timeModified - bool atomic - bool pathCreate -CODE: - CHECK(strEqZ(class, PACKAGE_NAME_LIBC "::StorageWrite")); - - RETVAL = storageWriteMove( - storageNewWriteP( - storage, file, .modeFile = mode, .user = user, .group = group, .timeModified = (time_t)timeModified, - .noCreatePath = storageFeature(storage, storageFeaturePath) ? !pathCreate : false, .noSyncPath = !atomic, - .noAtomic = !atomic), - memContextPrior()); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -filterAdd(self, filter, param) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageWrite self - const String *filter = STR_NEW_SV($arg); - const String *param = STR_NEW_SV($arg); -CODE: - IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(self)); - storageFilterXsAdd(filterGroup, filter, param); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -open(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageWrite self -CODE: - ioWriteOpen(storageWriteIo(self)); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -UV -write(self, buffer) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageWrite self - const Buffer *buffer = BUF_CONST_SV($arg); -CODE: - ioWrite(storageWriteIo(self), buffer); - RETVAL = bufUsed(buffer); -OUTPUT: - RETVAL -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -close(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageWrite self -CODE: - ioWriteClose(storageWriteIo(self)); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); - -#################################################################################################################################### -void -DESTROY(self) -PREINIT: - MEM_CONTEXT_XS_TEMP_BEGIN() - { -INPUT: - pgBackRest::LibC::StorageWrite self -CODE: - storageWriteFree(self); -CLEANUP: - } - MEM_CONTEXT_XS_TEMP_END(); diff --git a/libc/xs/storage/storageWrite.xsh b/libc/xs/storage/storageWrite.xsh deleted file mode 100644 index 5d4a3fd29..000000000 --- a/libc/xs/storage/storageWrite.xsh +++ /dev/null @@ -1,4 +0,0 @@ -/*********************************************************************************************************************************** -Storage Write XS Header -***********************************************************************************************************************************/ -typedef StorageWrite *pgBackRest__LibC__StorageWrite; diff --git a/test/Vagrantfile b/test/Vagrantfile index e804ef4a7..23d4ec177 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -91,7 +91,7 @@ Vagrant.configure(2) do |config| #--------------------------------------------------------------------------------------------------------------------------- echo 'Build VM Images' && date rm -rf /backrest/test/.vagrant/docker/* - rm -rf /backrest/test/.vagrant/libc/* + rm -rf /backrest/test/.vagrant/bin/* rm -rf /backrest/test/.vagrant/package/* sudo su - vagrant -c '/backrest/test/test.pl --vm-build' diff --git a/test/code-count/file-type.yaml b/test/code-count/file-type.yaml index 60b41d438..b879b2478 100644 --- a/test/code-count/file-type.yaml +++ b/test/code-count/file-type.yaml @@ -239,14 +239,6 @@ lib/pgBackRest/InfoCommon.pm: class: test/harness type: perl -lib/pgBackRest/LibC.pm: - class: test/harness - type: perl - -lib/pgBackRest/LibCAuto.pm: - class: test/harness/auto - type: perl - lib/pgBackRest/Manifest.pm: class: test/harness type: perl @@ -279,82 +271,6 @@ lib/pgBackRest/Version.pm: class: test/harness type: perl -libc/LibC.h: - class: test/harness - type: c/h - -libc/LibC.xs: - class: test/harness - type: xs - -libc/Makefile.PL: - class: build - type: perl - -libc/build/lib/pgBackRestLibC/Build.pm: - class: build - type: perl - -libc/xs/config/config.xs: - class: test/harness - type: xs - -libc/xs/config/configTest.xs: - class: test/harness - type: xs - -libc/xs/config/configTest.xsh: - class: test/harness - type: c/h - -libc/xs/config/define.xs: - class: test/harness - type: xs - -libc/xs/crypto/hash.xs: - class: test/harness - type: xs - -libc/xs/crypto/hash.xsh: - class: test/harness - type: c/h - -libc/xs/postgres/client.xs: - class: test/harness - type: xs - -libc/xs/postgres/client.xsh: - class: test/harness - type: c/h - -libc/xs/postgres/pageChecksum.xs: - class: test/harness - type: xs - -libc/xs/storage/storage.xs: - class: test/harness - type: xs - -libc/xs/storage/storage.xsh: - class: test/harness - type: c/h - -libc/xs/storage/storageRead.xs: - class: test/harness - type: xs - -libc/xs/storage/storageRead.xsh: - class: test/harness - type: c/h - -libc/xs/storage/storageWrite.xs: - class: test/harness - type: xs - -libc/xs/storage/storageWrite.xsh: - class: test/harness - type: c/h - src/Makefile.in: class: build type: make diff --git a/test/lib/pgBackRestTest/Common/BuildTest.pm b/test/lib/pgBackRestTest/Common/BuildTest.pm index 58ca77c91..d5ebbd728 100644 --- a/test/lib/pgBackRestTest/Common/BuildTest.pm +++ b/test/lib/pgBackRestTest/Common/BuildTest.pm @@ -113,7 +113,7 @@ sub buildDependencyTree # Only process non-auto files if ($strFile =~ /^[A-Za-z0-9\/]+\.(c|h)$/) { - buildDependencyTreeSub($oStorage, $rhDependencyTree, $strFile, true, undef, ['src', 'libc']); + buildDependencyTreeSub($oStorage, $rhDependencyTree, $strFile, true, undef, ['src']); } } @@ -229,8 +229,7 @@ sub buildMakefileObjectCompile foreach my $strInclude (@{$rhDependencyTree->{$strFile}{include}}) { - $strDepend .= - ' ' . ($oStorage->exists("src/${strInclude}") || $strInclude eq BUILD_AUTO_H ? '' : '../libc/') . $strInclude; + $strDepend .= " ${strInclude}"; } $strMakefile .= @@ -288,27 +287,5 @@ sub buildMakefile push @EXPORT, qw(buildMakefile); -#################################################################################################################################### -# Load the C library and check pointer size -#################################################################################################################################### -sub buildLoadLibC -{ - # Load the module dynamically - require pgBackRest::LibC; - pgBackRest::LibC->import(qw(:debug)); - - # Load shared object - require XSLoader; - XSLoader::load('pgBackRest::LibC', '999'); - - # Do a basic test to make sure it installed correctly - if (libcUvSize() != 8) - { - confess &log(ERROR, 'UVSIZE in test library does not equal 8'); - } -} - -push @EXPORT, qw(buildLoadLibC); - 1; diff --git a/test/lib/pgBackRestTest/Common/CodeCountTest.pm b/test/lib/pgBackRestTest/Common/CodeCountTest.pm index 490699ea3..e4959d1ed 100644 --- a/test/lib/pgBackRestTest/Common/CodeCountTest.pm +++ b/test/lib/pgBackRestTest/Common/CodeCountTest.pm @@ -55,7 +55,6 @@ sub codeCountScan $strFile =~ '\.eps$' || $strFile =~ '\.cache$' || $strFile =~ '^doc/site/' || - $strFile eq 'libc/typemap' || $strFile eq 'test/Vagrantfile' || $strFile =~ '^test/\.vagrant/' || $strFile =~ '^test/certificate/' || @@ -83,13 +82,12 @@ sub codeCountScan { $strClass = 'doc/core'; } - elsif ($strFile =~ '^build/' || $strFile =~ '^libc/build/' || $strFile eq 'libc/Makefile.PL' || - $strFile eq 'src/Makefile.in' || $strFile eq 'src/configure' || $strFile eq 'src/configure.ac') + elsif ($strFile =~ '^build/' || $strFile eq 'src/Makefile.in' || $strFile eq 'src/configure' || + $strFile eq 'src/configure.ac') { $strClass = 'build'; } - elsif ($strFile =~ '^test/lib/pgBackRestTest/Module/' || $strFile =~ '^test/src/module/' || - $strFile =~ '^libc/t/') + elsif ($strFile =~ '^test/lib/pgBackRestTest/Module/' || $strFile =~ '^test/src/module/') { $strClass = 'test/module'; } diff --git a/test/lib/pgBackRestTest/Common/FileTest.pm b/test/lib/pgBackRestTest/Common/FileTest.pm index 702e86af8..c621d4950 100644 --- a/test/lib/pgBackRestTest/Common/FileTest.pm +++ b/test/lib/pgBackRestTest/Common/FileTest.pm @@ -25,7 +25,6 @@ use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::String; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::Manifest; use pgBackRest::Storage::Base; @@ -154,7 +153,7 @@ sub forceStorageMode ( $strOperation, $oStorage, - $strPathExp, + $strPath, $strMode, $bRecurse ) = @@ -162,15 +161,15 @@ sub forceStorageMode ( __PACKAGE__ . '::forceStorageMode', \@_, {name => 'oStorage'}, - {name => 'strPathExp'}, + {name => 'strPath'}, {name => 'strMode'}, {name => 'bRecurse', optional => true, default => false}, ); - # Mode commands are ignored on S3 - if (!$oStorage->can('type') || $oStorage->type() ne STORAGE_S3) + # Mode commands are ignored on object storage + if ($oStorage->type() ne STORAGE_OBJECT) { - executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp)); + executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} ${strPath}"); } # Return from function and log return values if any @@ -189,55 +188,48 @@ sub forceStorageMove ( $strOperation, $oStorage, - $strSourcePathExp, - $strDestinationPathExp, + $strSourcePath, + $strDestinationPath, $bRecurse, ) = logDebugParam ( __PACKAGE__ . '->forceStorageMove', \@_, {name => 'oStorage'}, - {name => 'strSourcePathExp'}, - {name => 'strDestinationPathExp'}, + {name => 'strSourcePath'}, + {name => 'strDestinationPath'}, {name => 'bRecurse', optional => true, default => true}, ); - # If S3 then use storage commands to remove - if ($oStorage->type() eq STORAGE_S3) + # If object storage then use storage commands to remove + if ($oStorage->type() eq STORAGE_OBJECT) { if ($bRecurse) { - my $rhManifest = $oStorage->manifest($strSourcePathExp); + my $rhManifest = $oStorage->manifest($strSourcePath); foreach my $strName (sort(keys(%{$rhManifest}))) { if ($rhManifest->{$strName}{type} eq 'f') { $oStorage->put( - new pgBackRest::Storage::StorageWrite( - $oStorage, - pgBackRest::LibC::StorageWrite->new( - $oStorage->{oStorageC}, "${strDestinationPathExp}/${strName}", 0, undef, undef, 0, true, false)), - ${$oStorage->get( - new pgBackRest::Storage::StorageRead( - $oStorage, - pgBackRest::LibC::StorageRead->new( - $oStorage->{oStorageC}, "${strSourcePathExp}/${strName}", false)))}); - - $oStorage->remove("${strSourcePathExp}/${strName}"); + "${strDestinationPath}/${strName}", ${$oStorage->get("${strSourcePath}/${strName}", {bRaw => true})}, + {bRaw => true}); } } + + $oStorage->pathRemove($strSourcePath, {bRecurse => true}); } else { - $oStorage->put($strDestinationPathExp, ${$oStorage->get($strSourcePathExp)}); - $oStorage->remove($strSourcePathExp); + $oStorage->put($strDestinationPath, ${$oStorage->get($strSourcePath)}); + $oStorage->remove($strSourcePath); } } # Else remove using filesystem commands else { - executeTest('mv ' . $oStorage->pathGet($strSourcePathExp) . ' ' . $oStorage->pathGet($strDestinationPathExp)); + executeTest("mv ${strSourcePath} ${strDestinationPath}"); } # Return from function and log return values if any @@ -246,41 +238,6 @@ sub forceStorageMove push(@EXPORT, qw(forceStorageMove)); -#################################################################################################################################### -# forceStorageOwner - force ownership on a file or path -#################################################################################################################################### -sub forceStorageOwner -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strPathExp, - $strOwner, - $bRecurse - ) = - logDebugParam - ( - __PACKAGE__ . '::forceStorageOwner', \@_, - {name => 'oStorage'}, - {name => 'strPathExp'}, - {name => 'strOwner'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - # Owner commands are ignored on S3 - if ($oStorage->type() ne STORAGE_S3) - { - executeTest('chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp)); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageOwner)); - #################################################################################################################################### # forceStorageRemove - force remove a file or path from storage #################################################################################################################################### @@ -291,34 +248,25 @@ sub forceStorageRemove ( $strOperation, $oStorage, - $strPathExp, + $strPath, $bRecurse ) = logDebugParam ( __PACKAGE__ . '->forceStorageRemove', \@_, {name => 'oStorage'}, - {name => 'strPathExp'}, + {name => 'strPath'}, {name => 'bRecurse', optional => true, default => false}, ); - # If S3 then use storage commands to remove - if ($oStorage->type() eq STORAGE_S3) + # If object storage then use storage commands to remove + if ($oStorage->type() eq STORAGE_OBJECT) { - my $oInfo = $oStorage->info($strPathExp, {bIgnoreMissing => true}); - - if (defined($oInfo) && $oInfo->{type} eq 'f') - { - $oStorage->remove($strPathExp); - } - else - { - $oStorage->pathRemove($strPathExp, {bRecurse => true}); - } + $oStorage->pathRemove($strPath, {bRecurse => true}); } else { - executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp)); + executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $strPath); } # Return from function and log return values if any diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index fa21efc6d..b77bf3044 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -221,13 +221,11 @@ sub run if (!$rhBuildInit->{$self->{oTest}->{&TEST_VM}}{$self->{iVmIdx}}) { executeTest( - 'rsync -rt --delete --exclude=*.o --exclude=test.c --exclude=test.gcno --exclude=LibC.h --exclude=xs' . + 'rsync -rt --delete --exclude=*.o --exclude=test.c --exclude=test.gcno ' . ' --exclude=test --exclude=buildflags --exclude=testflags --exclude=harnessflags' . ' --exclude=build.auto.h --exclude=build.auto.h.in --exclude=Makefile --exclude=Makefile.in' . ' --exclude=configure --exclude=configure.ac' . " $self->{strBackRestBase}/src/ $self->{strGCovPath} && " . - "rsync -t $self->{strBackRestBase}/libc/LibC.h $self->{strGCovPath} && " . - "rsync -rt --delete $self->{strBackRestBase}/libc/xs/ $self->{strGCovPath}/xs && " . "rsync -rt --delete --exclude=*.o $self->{strBackRestBase}/test/src/ $self->{strGCovPath}/test"); } @@ -794,7 +792,6 @@ sub jobInstallC my $oVm = vmGet(); my $strBuildPath = "${strBasePath}/test/.vagrant/bin/${strVm}"; - my $strBuildLibCPath = "${strBuildPath}/libc"; my $strBuildBinPath = "${strBuildPath}/src"; executeTest( diff --git a/test/lib/pgBackRestTest/Common/ListTest.pm b/test/lib/pgBackRestTest/Common/ListTest.pm index 1c37b2e6d..13c1c56ae 100644 --- a/test/lib/pgBackRestTest/Common/ListTest.pm +++ b/test/lib/pgBackRestTest/Common/ListTest.pm @@ -115,7 +115,7 @@ sub testListGet # Skip this test if it is integration and vm=none next if ($strVm eq VM_NONE && $hTest->{&TESTDEF_TYPE} eq TESTDEF_INTEGRATION); - # Skip this test if it is not C and vm=none. Perl tests require libc which is not supported. + # Skip this test if it is not C and vm=none. Perl tests require Docker which is not supported. next if ($strVm eq VM_NONE && !$hTest->{&TESTDEF_C}); # Skip this test if a container is required and vm=none. diff --git a/test/lib/pgBackRestTest/Common/RunTest.pm b/test/lib/pgBackRestTest/Common/RunTest.pm index 4c9abf2dc..3cc09ca5a 100644 --- a/test/lib/pgBackRestTest/Common/RunTest.pm +++ b/test/lib/pgBackRestTest/Common/RunTest.pm @@ -114,7 +114,7 @@ sub process $self->{iVmId}, $self->{strBasePath}, $self->{strTestPath}, - $self->{strBackRestExeC}, + $self->{strBackRestExe}, $self->{strBackRestExeHelper}, $self->{strPgBinPath}, $self->{strPgVersion}, @@ -136,7 +136,7 @@ sub process {name => 'iVmId'}, {name => 'strBasePath'}, {name => 'strTestPath'}, - {name => 'strBackRestExeC'}, + {name => 'strBackRestExe'}, {name => 'strBackRestExeHelper'}, {name => 'strPgBinPath', required => false}, {name => 'strPgVersion', required => false}, @@ -159,9 +159,6 @@ sub process $oStorage = new pgBackRestTest::Common::Storage( $self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); - # Generate backrest exe - $self->{strBackRestExe} = defined($self->{strBackRestExeC}) ? $self->{strBackRestExeC} : $self->{strBackRestExeHelper}; - projectBinSet($self->{strBackRestExe}); # Init, run, and end the test(s) @@ -545,6 +542,7 @@ push(@EXPORT, qw(storageTest)); #################################################################################################################################### sub archBits {return vmArchBits(shift->{strVm})} sub backrestExe {return shift->{strBackRestExe}} +sub backrestExeHelper {return shift->{strBackRestExeHelper}} sub basePath {return shift->{strBasePath}} sub dataPath {return shift->basePath() . '/test/data'} sub doCleanup {return shift->{bCleanup}} diff --git a/test/lib/pgBackRestTest/Common/Storage.pm b/test/lib/pgBackRestTest/Common/Storage.pm index a15b1d0a3..c95aee6f0 100644 --- a/test/lib/pgBackRestTest/Common/Storage.pm +++ b/test/lib/pgBackRestTest/Common/Storage.pm @@ -69,6 +69,85 @@ sub new ); } +#################################################################################################################################### +# Copy a file. If special encryption settings are required, then the file objects from openRead/openWrite must be passed instead of +# file names. +#################################################################################################################################### +sub copy +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xSourceFile, + $xDestinationFile, + $bSourceOpen, + ) = + logDebugParam + ( + __PACKAGE__ . '->copy', \@_, + {name => 'xSourceFile', required => false}, + {name => 'xDestinationFile'}, + {name => 'bSourceOpen', optional => true, default => false}, + ); + + # Is source/destination an IO object or a file expression? + my $oSourceFileIo = defined($xSourceFile) ? (ref($xSourceFile) ? $xSourceFile : $self->openRead($xSourceFile)) : undef; + + # Does the source file exist? + my $bResult = false; + + # Copy if the source file exists + if (defined($oSourceFileIo)) + { + my $oDestinationFileIo = ref($xDestinationFile) ? $xDestinationFile : $self->openWrite($xDestinationFile); + + # Use C copy if source and destination are C objects + if (defined($oSourceFileIo->{oStorageCRead}) && defined($oDestinationFileIo->{oStorageCWrite})) + { + $bResult = $self->{oStorageC}->copy( + $oSourceFileIo->{oStorageCRead}, $oDestinationFileIo->{oStorageCWrite}) ? true : false; + } + else + { + # Open the source file if it is a C object + $bResult = defined($oSourceFileIo->{oStorageCRead}) ? ($bSourceOpen || $oSourceFileIo->open()) : true; + + if ($bResult) + { + # Open the destination file if it is a C object + if (defined($oDestinationFileIo->{oStorageCWrite})) + { + $oDestinationFileIo->open(); + } + + # Copy the data + do + { + # Read data + my $tBuffer = ''; + + $oSourceFileIo->read(\$tBuffer, $self->{lBufferMax}); + $oDestinationFileIo->write(\$tBuffer); + } + while (!$oSourceFileIo->eof()); + + # Close files + $oSourceFileIo->close(); + $oDestinationFileIo->close(); + } + } + } + + return logDebugReturn + ( + $strOperation, + {name => 'bResult', value => $bResult, trace => true}, + ); +} + #################################################################################################################################### # exists - check if file exists #################################################################################################################################### @@ -99,6 +178,62 @@ sub exists ); } +#################################################################################################################################### +# get - reads a buffer from storage all at once +#################################################################################################################################### +sub get +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xFile, + ) = + logDebugParam + ( + __PACKAGE__ . '->get', \@_, + {name => 'xFile', required => false, trace => true}, + ); + + # Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or + # if the repo has a user passphrase defined - else pass undef + my $oFileIo = defined($xFile) ? (ref($xFile) ? $xFile : $self->openRead($xFile)) : undef; + + # Read only if there is something to read from + my $tContent; + my $lSize = 0; + + if (defined($oFileIo)) + { + my $lSizeRead; + + do + { + $lSizeRead = $oFileIo->read(\$tContent, $self->{lBufferMax}); + $lSize += $lSizeRead; + } + while ($lSizeRead != 0); + + # Close the file + $oFileIo->close(); + + # If nothing was read then set to undef + if ($lSize == 0) + { + $tContent = undef; + } + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'rtContent', value => defined($oFileIo) ? \$tContent : undef, trace => true}, + ); +} + #################################################################################################################################### # info - get information for path/file #################################################################################################################################### @@ -595,6 +730,56 @@ sub pathSync return logDebugReturn($strOperation); } +#################################################################################################################################### +# put - writes a buffer out to storage all at once +#################################################################################################################################### +sub put +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $xFile, + $xContent, + ) = + logDebugParam + ( + __PACKAGE__ . '->put', \@_, + {name => 'xFile', trace => true}, + {name => 'xContent', required => false, trace => true}, + ); + + # Is this an IO object or a file expression? If file expression, then open the file and pass passphrase if one is defined or if + # the repo has a user passphrase defined - else pass undef + my $oFileIo = ref($xFile) ? $xFile : $self->openWrite($xFile); + + # Determine size of content + my $lSize = defined($xContent) ? length(ref($xContent) ? $$xContent : $xContent) : 0; + + # Write only if there is something to write + if ($lSize > 0) + { + $oFileIo->write(ref($xContent) ? $xContent : \$xContent); + } + # Else open the file so a zero length file is created (since file is not opened until first write) + else + { + $oFileIo->open(); + } + + # Close the file + $oFileIo->close(); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'lSize', value => $lSize, trace => true}, + ); +} + #################################################################################################################################### # remove - remove path/file #################################################################################################################################### @@ -647,7 +832,6 @@ sub remove #################################################################################################################################### sub pathBase {shift->{strPathBase}} sub driver {shift->{oDriver}} -sub cipherType {undef} -sub cipherPassUser {undef} +sub type {shift->{oDriver}->type()} 1; diff --git a/test/lib/pgBackRestTest/Common/StoragePosix.pm b/test/lib/pgBackRestTest/Common/StoragePosix.pm index 8c788a4a7..913c619f7 100644 --- a/test/lib/pgBackRestTest/Common/StoragePosix.pm +++ b/test/lib/pgBackRestTest/Common/StoragePosix.pm @@ -978,5 +978,6 @@ sub remove sub className {STORAGE_POSIX_DRIVER} sub tempExtension {shift->{strTempExtension}} sub tempExtensionSet {my $self = shift; $self->{strTempExtension} = shift} +sub type {STORAGE_POSIX} 1; diff --git a/test/lib/pgBackRestTest/Common/VmTest.pm b/test/lib/pgBackRestTest/Common/VmTest.pm index 121016523..1b995ae20 100644 --- a/test/lib/pgBackRestTest/Common/VmTest.pm +++ b/test/lib/pgBackRestTest/Common/VmTest.pm @@ -509,7 +509,7 @@ sub vmValid if (!defined($oyVm->{$strVm})) { - confess &log(ERROR, "no definition for vm '${strVm}'", ERROR_OPTION_INVALID_VALUE); + confess &log(ERROR, "no definition for vm '${strVm}'"); } } diff --git a/test/lib/pgBackRestTest/Env/ConfigEnvTest.pm b/test/lib/pgBackRestTest/Env/ConfigEnvTest.pm deleted file mode 100644 index d4e14913b..000000000 --- a/test/lib/pgBackRestTest/Env/ConfigEnvTest.pm +++ /dev/null @@ -1,191 +0,0 @@ -#################################################################################################################################### -# ConfigCommonTest.pm - Common code for Config unit tests -#################################################################################################################################### -package pgBackRestTest::Env::ConfigEnvTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Getopt::Long qw(GetOptions); - -use pgBackRest::Common::Exception; -use pgBackRest::Common::Log; -use pgBackRest::Config::Config; -use pgBackRest::LibC qw(:test); -use pgBackRest::Version; - -use pgBackRestTest::Common::RunTest; - -use constant CONFIGENVTEST => 'ConfigEnvTest'; - -#################################################################################################################################### -# Is the option secure? -#################################################################################################################################### -sub optionTestSecure -{ - my $self = shift; - my $strOption = shift; - - return (cfgDefOptionSecure(cfgOptionId($strOption)) ? true : false); -} - -sub optionTestSet -{ - my $self = shift; - my $iOptionId = shift; - my $strValue = shift; - - $self->{&CONFIGENVTEST}{option}{cfgOptionName($iOptionId)} = $strValue; -} - -sub optionTestSetBool -{ - my $self = shift; - my $iOptionId = shift; - my $bValue = shift; - - $self->{&CONFIGENVTEST}{boolean}{cfgOptionName($iOptionId)} = defined($bValue) ? $bValue : true; -} - -sub optionTestClear -{ - my $self = shift; - my $iOptionId = shift; - - delete($self->{&CONFIGENVTEST}{option}{cfgOptionName($iOptionId)}); - delete($self->{&CONFIGENVTEST}{boolean}{cfgOptionName($iOptionId)}); -} - -sub configTestClear -{ - my $self = shift; - - my $rhConfig = $self->{&CONFIGENVTEST}; - - delete($self->{&CONFIGENVTEST}); - - return $rhConfig; -} - -sub configTestSet -{ - my $self = shift; - my $rhConfig = shift; - - $self->{&CONFIGENVTEST} = $rhConfig; -} - -#################################################################################################################################### -# Write all secure options to a config file -#################################################################################################################################### -sub configFileWrite -{ - my $self = shift; - my $strConfigFile = shift; - my $rhConfig = shift; - - my $strConfig = "[global]\n"; - - if (defined($rhConfig->{boolean})) - { - foreach my $strOption (sort(keys(%{$rhConfig->{boolean}}))) - { - if ($self->optionTestSecure($strOption)) - { - $strConfig .= "${strOption}=" . ($rhConfig->{boolean}{$strOption} ? 'y' : 'n') . "\n"; - } - } - } - - if (defined($rhConfig->{option})) - { - foreach my $strOption (sort(keys(%{$rhConfig->{option}}))) - { - if ($self->optionTestSecure($strOption)) - { - $strConfig .= "${strOption}=$rhConfig->{option}{$strOption}\n"; - } - } - } - - storageTest()->put($strConfigFile, $strConfig); -} - -#################################################################################################################################### -# Write all non-secure options to the command line -#################################################################################################################################### -sub commandTestWrite -{ - my $self = shift; - my $strCommand = shift; - my $strConfigFile = shift; - my $rhConfig = shift; - - my @szyParam = (); - - # Add boolean options - if (defined($rhConfig->{boolean})) - { - foreach my $strOption (sort(keys(%{$rhConfig->{boolean}}))) - { - if (!$self->optionTestSecure($strOption)) - { - if ($rhConfig->{boolean}{$strOption}) - { - push(@szyParam, "--${strOption}"); - } - else - { - push(@szyParam, "--no-${strOption}"); - } - } - } - } - - # Add non-boolean options - if (defined($rhConfig->{option})) - { - foreach my $strOption (sort(keys(%{$rhConfig->{option}}))) - { - if (!$self->optionTestSecure($strOption)) - { - push(@szyParam, "--${strOption}=$rhConfig->{option}{$strOption}"); - } - } - } - - # Add config file - push(@szyParam, '--' . cfgOptionName(CFGOPT_CONFIG) . "=${strConfigFile}"); - - # Add command - push(@szyParam, $strCommand); - - return @szyParam; -} - -#################################################################################################################################### -# Load the configuration -#################################################################################################################################### -sub configTestLoad -{ - my $self = shift; - my $iCommandId = shift; - - # A config file is required to store secure options before they can be parsed - my $strConfigFile = $self->testPath() . '/pgbackrest.test.conf'; - $self->configFileWrite($strConfigFile, $self->{&CONFIGENVTEST}); - - my @stryArg = $self->commandTestWrite(cfgCommandName($iCommandId), $strConfigFile, $self->{&CONFIGENVTEST}); - my $strConfigJson = cfgParseTest(projectBin(), join('|', @stryArg)); - $self->testResult( - sub {configLoad(false, projectBin(), cfgCommandName($iCommandId), \$strConfigJson)}, - true, 'config load: ' . join(" ", @stryArg)); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm b/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm index 3d11fd3ce..9d4312dd2 100644 --- a/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm +++ b/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm @@ -20,10 +20,8 @@ use pgBackRest::Backup::Info; use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; -use pgBackRest::Config::Config; use pgBackRest::DbVersion; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; use pgBackRest::Version; @@ -50,6 +48,7 @@ sub new $self->{oHostBackup}, $self->{strBackRestExe}, $self->{oStorageRepo}, + $self->{strPgPath}, $self->{oLogTest}, $self->{oRunTest}, ) = @@ -59,6 +58,7 @@ sub new {name => 'oHostBackup', required => false, trace => true}, {name => 'strBackRestExe', trace => true}, {name => 'oStorageRepo', trace => true}, + {name => 'strPgPath', trace => true}, {name => 'oLogTest', required => false, trace => true}, {name => 'oRunTest', required => false, trace => true}, ); @@ -124,7 +124,7 @@ sub info logDebugParam ( __PACKAGE__ . '->info', \@_, - {name => 'strDbPath', default => cfgOption(CFGOPT_PG_PATH)} + {name => 'strDbPath', default => $self->{strPgPath}} ); # Open the control file and read system id and versions @@ -168,8 +168,7 @@ sub info ERROR, 'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} . ' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" . - 'HINT: is this version of PostgreSQL supported?', - ERROR_VERSION_NOT_SUPPORTED); + 'HINT: is this version of PostgreSQL supported?'); } # Return from function and log return values if any @@ -210,7 +209,6 @@ sub stanzaSet my $oStanza = {}; my $oArchiveInfo = {}; my $oBackupInfo = {}; - my $bEncrypted = defined($self->{oStorageRepo}->cipherType()); my $iArchiveDbId = 1; my $iBackupDbId = 1; @@ -218,26 +216,21 @@ sub stanzaSet if (!$bStanzaUpgrade) { $oArchiveInfo = - new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE), false, - {bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_ARCHIVE : undef}); + new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath(), false, + {bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef}); $oBackupInfo = - new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP), false, - {bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_MANIFEST : undef}); + new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath(), false, + {bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef}); } # Else get the info data from disk else { $oArchiveInfo = - new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE), - {strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_ARCHIVE : undef}); + new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath(), + {strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef}); $oBackupInfo = - new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP), - {strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_MANIFEST : undef}); - } - - if (cfgOption(CFGOPT_ONLINE)) - { - confess &log(ERROR, "this function may not be used for online tests"); + new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath(), + {strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef}); } # Get the database info for the stanza @@ -258,8 +251,8 @@ sub stanzaSet $oBackupInfo->save(); # Get the archive and directory paths for the stanza - $$oStanza{strArchiveClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE) . '/' . ($oArchiveInfo->archiveId()); - $$oStanza{strBackupClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP); + $$oStanza{strArchiveClusterPath} = $self->{oHostBackup}->repoArchivePath($oArchiveInfo->archiveId()); + $$oStanza{strBackupClusterPath} = $self->{oHostBackup}->repoBackupPath(); $self->{oStanzaHash}{$strStanza} = $oStanza; @@ -291,14 +284,12 @@ sub stanzaCreate my $strDbVersionTemp = $strDbVersion; $strDbVersionTemp =~ s/\.//; - my $strDbPath = cfgOption(CFGOPT_PG_PATH); - # Create the test path for pg_control - storageTest()->pathCreate(($strDbPath . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true}); + storageTest()->pathCreate(($self->{strPgPath} . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true}); # Generate pg_control for stanza-create - $self->controlGenerate($strDbPath, $strDbVersion); - executeTest('chmod 600 ' . $strDbPath . '/' . DB_FILE_PGCONTROL); + $self->controlGenerate($self->{strPgPath}, $strDbVersion); + executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL); # Create the stanza and set the local stanza object $self->stanzaSet($strStanza, $strDbVersion, false); @@ -332,11 +323,11 @@ sub stanzaUpgrade $strDbVersionTemp =~ s/\.//; # Remove pg_control - storageTest()->remove(cfgOption(CFGOPT_PG_PATH) . '/' . DB_FILE_PGCONTROL); + storageTest()->remove($self->{strPgPath} . '/' . DB_FILE_PGCONTROL); # Copy pg_control for stanza-upgrade - $self->controlGenerate(cfgOption(CFGOPT_PG_PATH), $strDbVersion); - executeTest('chmod 600 ' . cfgOption(CFGOPT_PG_PATH) . '/' . DB_FILE_PGCONTROL); + $self->controlGenerate($self->{strPgPath}, $strDbVersion); + executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL); $self->stanzaSet($strStanza, $strDbVersion, true); @@ -393,7 +384,7 @@ sub backupCreate # Get passphrase (returns undefined if repo not encrypted) to access the manifest my $strCipherPassManifest = - (new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP)))->cipherPassSub(); + (new pgBackRest::Backup::Info($self->{oHostBackup}->repoBackupPath()))->cipherPassSub(); my $strCipherPassBackupSet; # If repo is encrypted then get passphrase for accessing the backup files from the last manifest if it exists provide one @@ -540,7 +531,7 @@ sub archiveCreate # Get passphrase (returns undefined if repo not encrypted) to access the archive files my $strCipherPass = - (new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE)))->cipherPassSub(); + (new pgBackRest::Archive::Info($self->{oHostBackup}->repoArchivePath()))->cipherPassSub(); push(my @stryArchive, $strArchive); @@ -604,7 +595,7 @@ sub supplementalLog join("\n", grep(!/^backup\.info.*$/i, storageRepo()->list("backup/${strStanza}")))); # Output archive manifest - my $rhManifest = storageRepo()->manifest(STORAGE_REPO_ARCHIVE); + my $rhManifest = storageRepo()->manifest($self->{oHostBackup}->repoArchivePath()); my $strManifest; my $strPrefix = ''; @@ -615,7 +606,7 @@ sub supplementalLog if ($rhManifest->{$strEntry}->{type} eq 'd') { - $strEntry = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE) . ($strEntry eq '.' ? '' : "/${strEntry}"); + $strEntry = $self->{oHostBackup}->repoArchivePath($strEntry eq '.' ? undef : $strEntry); # &log(WARN, "DIR $strEntry"); $strManifest .= (defined($strManifest) ? "\n" : '') . "${strEntry}:\n"; @@ -670,10 +661,9 @@ sub process undef($$oStanza{strBackupDescription}); - my $strCommand = $self->{strBackRestExe} . - ' --' . cfgOptionName(CFGOPT_CONFIG) . '="' . $self->{oHostBackup}->backrestConfig() . '"' . - ' --' . cfgOptionName(CFGOPT_STANZA) . '=' . $strStanza . - ' --' . cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) . '=' . lc(DETAIL); + my $strCommand = + $self->{strBackRestExe} . ' --config="' . $self->{oHostBackup}->backrestConfig() . '"' . ' --stanza=' . $strStanza . + ' --log-level-console=' . lc(DETAIL); if (defined($iExpireFull)) { diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm index df78f91b5..6a8df571b 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm @@ -24,12 +24,11 @@ use pgBackRest::Backup::Info; use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; -use pgBackRest::Config::Config; use pgBackRest::DbVersion; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; -use pgBackRest::Version; use pgBackRest::Storage::Base; +use pgBackRest::Storage::Helper; +use pgBackRest::Version; use pgBackRestTest::Env::Host::HostBaseTest; use pgBackRestTest::Env::Host::HostS3Test; @@ -50,6 +49,49 @@ use constant HOST_PATH_REPO => 'repo'; use constant HOST_PROTOCOL_TIMEOUT => 10; push @EXPORT, qw(HOST_PROTOCOL_TIMEOUT); +#################################################################################################################################### +# Configuration constants +#################################################################################################################################### +use constant CFGDEF_SECTION_GLOBAL => 'global'; + push @EXPORT, qw(CFGDEF_SECTION_GLOBAL); +use constant CFGDEF_SECTION_STANZA => 'stanza'; + push @EXPORT, qw(CFGDEF_SECTION_STANZA); + +use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; + push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); +use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; + push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); +use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; + push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); + +use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc'; + push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC); + +use constant STORAGE_CIFS => 'cifs'; + push @EXPORT, qw(STORAGE_CIFS); +use constant STORAGE_S3 => 's3'; + push @EXPORT, qw(STORAGE_S3); + +use constant CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default'; + push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_DEFAULT); +use constant CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate'; + push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); +use constant CFGOPTVAL_RESTORE_TYPE_NAME => 'name'; + push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_NAME); +use constant CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve'; + push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_PRESERVE); +use constant CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby'; + push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_STANDBY); +use constant CFGOPTVAL_RESTORE_TYPE_TIME => 'time'; + push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_TIME); +use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid'; + push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_XID); + +use constant NONE => 'none'; + push @EXPORT, qw(NONE); +use constant GZ => 'gz'; + push @EXPORT, qw(GZ); + #################################################################################################################################### # new #################################################################################################################################### @@ -251,8 +293,8 @@ sub backupEnd { if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink()) { - my $hTablespaceManifest = storageRepo()->manifest( - STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC); + my $hTablespaceManifest = storageTest()->manifest( + $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)); # Remove . and .. delete($hTablespaceManifest->{'.'}); @@ -305,19 +347,17 @@ sub backupEnd } # Else there should not be a tablespace directory at all. This is only valid for storage that supports links. elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) && - storageTest()->pathExists( - storageRepo()->pathGet( - STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))) + storageTest()->pathExists($self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))) { confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory'); } } # Check that latest link exists unless repo links are disabled - my $strLatestLink = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST); + my $strLatestLink = $self->repoBackupPath(LINK_LATEST); my $bLatestLinkExists = storageRepo()->exists($strLatestLink); - if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq CFGOPTVAL_REPO_TYPE_POSIX) && $self->hasLink()) + if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq STORAGE_POSIX) && $self->hasLink()) { my $strLatestLinkDestination = readlink($strLatestLink); @@ -367,14 +407,12 @@ sub backupEnd if ($self->synthetic() && $bManifestCompare) { $self->{oLogTest}->supplementalAdd( - storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), undef, + $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), undef, ${storageRepo()->get( storageRepo()->openRead( - STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST, - {strCipherPass => $self->cipherPassManifest()}))}); + $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}))}); $self->{oLogTest}->supplementalAdd( - storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO), undef, - ${storageRepo->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)}); + $self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo->get($self->repoBackupPath(FILE_BACKUP_INFO))}); } } @@ -446,8 +484,7 @@ sub backupCompare ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup; my $oActualManifest = new pgBackRest::Manifest( - storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), - {strCipherPass => $self->cipherPassManifest()}); + $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}); ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} = $oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START); @@ -478,7 +515,7 @@ sub backupCompare # Determine repo size if compression or encryption is enabled my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}; - if ($strCompressType ne CFGOPTVAL_COMPRESS_TYPE_NONE || + if ($strCompressType ne NONE || (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}))) { @@ -486,9 +523,9 @@ sub backupCompare my $lRepoSize = $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ? $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) : - (storageRepo()->info(STORAGE_REPO_BACKUP . - "/${strBackup}/${strFileKey}" . - ($strCompressType eq CFGOPTVAL_COMPRESS_TYPE_NONE ? '' : ".${strCompressType}")))->{size}; + (storageRepo()->info( + $self->repoBackupPath("${strBackup}/${strFileKey}") . + ($strCompressType eq NONE ? '' : ".${strCompressType}")))->{size}; if (defined($lRepoSize) && $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE}) @@ -613,7 +650,7 @@ sub backupLast my $self = shift; my @stryBackup = storageRepo()->list( - STORAGE_REPO_BACKUP, {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'}); + $self->repoBackupPath(), {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'}); if (!defined($stryBackup[0])) { @@ -780,13 +817,11 @@ sub stanzaCreate if (defined($self->{oLogTest}) && $self->synthetic()) { $self->{oLogTest}->supplementalAdd( - storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef, - ${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)}); + $self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo()->get($self->repoBackupPath(FILE_BACKUP_INFO))}); } # Get the passphrase for accessing the manifest file - $self->{strCipherPassManifest} = - (new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza())))->cipherPassSub(); + $self->{strCipherPassManifest} = (new pgBackRest::Backup::Info($self->repoBackupPath()))->cipherPassSub(); } if (storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE)) @@ -795,13 +830,12 @@ sub stanzaCreate if (defined($self->{oLogTest}) && $self->synthetic()) { $self->{oLogTest}->supplementalAdd( - storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef, - ${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)}); + $self->repoArchivePath(ARCHIVE_INFO_FILE), undef, ${storageRepo()->get($self->repoArchivePath(ARCHIVE_INFO_FILE))}); } # Get the passphrase for accessing the archived files $self->{strCipherPassArchive} = - (new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza())))->cipherPassSub(); + (new pgBackRest::Archive::Info($self->repoArchivePath()))->cipherPassSub(); } # Return from function and log return values if any @@ -848,16 +882,14 @@ sub stanzaUpgrade storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO)) { $self->{oLogTest}->supplementalAdd( - storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef, - ${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)}); + $self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo()->get($self->repoBackupPath(FILE_BACKUP_INFO))}); } if (defined($self->{oLogTest}) && $self->synthetic() && storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE)) { $self->{oLogTest}->supplementalAdd( - storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef, - ${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)}); + $self->repoArchivePath(ARCHIVE_INFO_FILE), undef, ${storageRepo()->get($self->repoArchivePath(ARCHIVE_INFO_FILE))}); } # Return from function and log return values if any @@ -1009,65 +1041,64 @@ sub configCreate # General options # ------------------------------------------------------------------------------------------------------------------------------ - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE)} = lc(DETAIL); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_FILE)} = testRunGet()->logLevelTestFile(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_LEVEL_STDERR)} = lc(OFF); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_SUBPROCESS)} = + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-console'} = lc(DETAIL); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-file'} = testRunGet()->logLevelTestFile(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-stderr'} = lc(OFF); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-subprocess'} = testRunGet()->logLevelTestFile() eq lc(OFF) ? 'n' : 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_TIMESTAMP)} = 'n'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_BUFFER_SIZE)} = '64k'; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-timestamp'} = 'n'; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'buffer-size'} = '64k'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_PATH)} = $self->logPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOCK_PATH)} = $self->lockPath(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_PROTOCOL_TIMEOUT)} = 60; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_DB_TIMEOUT)} = 45; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'protocol-timeout'} = 60; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'db-timeout'} = 45; # Set to make sure that changing the default works and to speed compression for testing - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_LEVEL)} = 3; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level'} = 3; # Only set network compress level if there is more than one host if ($oHostBackup != $oHostDbMaster) { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_LEVEL_NETWORK)} = 1; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level-network'} = 1; } - if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne CFGOPTVAL_COMPRESS_TYPE_GZ) + if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne 'gz') { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_TYPE)} = $oParam->{strCompressType}; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-type'} = $oParam->{strCompressType}; } if ($self->isHostBackup()) { if ($self->repoEncrypt()) { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_CIPHER_TYPE)} = - CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_CIPHER_PASS)} = 'x'; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-type'} = CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-pass'} = 'x'; } - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_PATH)} = $self->repoPath(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath(); # S3 settings if ($oParam->{bS3}) { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_TYPE)} = CFGOPTVAL_REPO_TYPE_S3; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_KEY)} = HOST_S3_ACCESS_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_KEY_SECRET)} = HOST_S3_ACCESS_SECRET_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_BUCKET)} = HOST_S3_BUCKET; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_ENDPOINT)} = HOST_S3_ENDPOINT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_S3_REGION)} = HOST_S3_REGION; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = STORAGE_S3; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key'} = HOST_S3_ACCESS_KEY; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key-secret'} = HOST_S3_ACCESS_SECRET_KEY; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-bucket'} = HOST_S3_BUCKET; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-endpoint'} = HOST_S3_ENDPOINT; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-region'} = HOST_S3_REGION; $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'} = 'n'; } if (defined($$oParam{bHardlink}) && $$oParam{bHardlink}) { $self->{bHardLink} = true; - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_REPO_HARDLINK)} = 'y'; + $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'repo1-s3-hardlink'} = 'y'; } - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_ARCHIVE_COPY)} = 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . cfgCommandName(CFGCMD_BACKUP)}{cfgOptionName(CFGOPT_START_FAST)} = 'y'; + $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'archive-copy'} = 'y'; + $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y'; } # Host specific options @@ -1087,53 +1118,44 @@ sub configCreate if ($self->nameTest(HOST_BACKUP)) { - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST)} = $oHostDb1->nameGet(); - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_USER)} = $oHostDb1->userGet(); - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_CMD)} = $oHostDb1->backrestExe(); - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_HOST_CONFIG)} = $oHostDb1->backrestConfig(); + $oParamHash{$strStanza}{'pg1-host'} = $oHostDb1->nameGet(); + $oParamHash{$strStanza}{'pg1-host-user'} = $oHostDb1->userGet(); + $oParamHash{$strStanza}{'pg1-host-cmd'} = $oHostDb1->backrestExe(); + $oParamHash{$strStanza}{'pg1-host-config'} = $oHostDb1->backrestConfig(); # Port can't be configured for a synthetic host if (!$self->synthetic()) { - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PORT)} = $oHostDb1->pgPort(); + $oParamHash{$strStanza}{'pg1-port'} = $oHostDb1->pgPort(); } } - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $oHostDb1->dbBasePath(); + $oParamHash{$strStanza}{'pg1-path'} = $oHostDb1->dbBasePath(); if (defined($oHostDb2)) { # Add an invalid replica to simulate more than one replica. A warning should be thrown when a stanza is created and a # valid replica should be chosen. - my $iInvalidReplica = 2; - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iInvalidReplica))} = BOGUS; - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iInvalidReplica))} = - $oHostDb2->userGet(); - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iInvalidReplica))} = - $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iInvalidReplica))} = - $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iInvalidReplica))} = - $oHostDb2->dbBasePath(); + $oParamHash{$strStanza}{"pg2-host"} = BOGUS; + $oParamHash{$strStanza}{"pg2-host-user"} = $oHostDb2->userGet(); + $oParamHash{$strStanza}{"pg2-host-cmd"} = $oHostDb2->backrestExe(); + $oParamHash{$strStanza}{"pg2-host-config"} = $oHostDb2->backrestConfig(); + $oParamHash{$strStanza}{"pg2-path"} = $oHostDb2->dbBasePath(); # Set a flag so we know there's a bogus host $self->{bBogusHost} = true; - # Set a valid replica to the last possible index to ensure skipping indexes does not make a difference. - my $iValidReplica = cfgOptionIndexTotal(CFGOPT_PG_PATH); - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, $iValidReplica))} = $oHostDb2->nameGet(); - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_USER, $iValidReplica))} = - $oHostDb2->userGet(); - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CMD, $iValidReplica))} = - $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST_CONFIG, $iValidReplica))} = - $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PATH, $iValidReplica))} = $oHostDb2->dbBasePath(); + # Set a valid replica to a higher index to ensure skipping indexes does not make a difference + $oParamHash{$strStanza}{"pg8-host"} = $oHostDb2->nameGet(); + $oParamHash{$strStanza}{"pg8-host-user"} = $oHostDb2->userGet(); + $oParamHash{$strStanza}{"pg8-host-cmd"} = $oHostDb2->backrestExe(); + $oParamHash{$strStanza}{"pg8-host-config"} = $oHostDb2->backrestConfig(); + $oParamHash{$strStanza}{"pg8-path"} = $oHostDb2->dbBasePath(); # Only test explicit ports on the backup server. This is so locally configured ports are also tested. if (!$self->synthetic() && $self->nameTest(HOST_BACKUP)) { - $oParamHash{$strStanza}{cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_PORT, $iValidReplica))} = $oHostDb2->pgPort(); + $oParamHash{$strStanza}{"pg8-port"} = $oHostDb2->pgPort(); } } } @@ -1141,32 +1163,31 @@ sub configCreate # If this is a database host if ($self->isHostDb()) { - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $self->dbBasePath(); + $oParamHash{$strStanza}{'pg1-path'} = $self->dbBasePath(); if (!$self->synthetic()) { - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_SOCKET_PATH)} = $self->pgSocketPath(); - $oParamHash{$strStanza}{cfgOptionName(CFGOPT_PG_PORT)} = $self->pgPort(); + $oParamHash{$strStanza}{'pg1-socket-path'} = $self->pgSocketPath(); + $oParamHash{$strStanza}{'pg1-port'} = $self->pgPort(); } if ($bArchiveAsync) { - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':' . - cfgCommandName(CFGCMD_ARCHIVE_PUSH)}{cfgOptionName(CFGOPT_ARCHIVE_ASYNC)} = 'y'; + $oParamHash{&CFGDEF_SECTION_GLOBAL . ':archive-push'}{'archive-async'} = 'y'; } - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_SPOOL_PATH)} = $self->spoolPath(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'spool-path'} = $self->spoolPath(); # If the the backup host is remote if (!$self->isHostBackup()) { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST)} = $oHostBackup->nameGet(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_USER)} = $oHostBackup->userGet(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_CMD)} = $oHostBackup->backrestExe(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_REPO_HOST_CONFIG)} = $oHostBackup->backrestConfig(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host'} = $oHostBackup->nameGet(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-user'} = $oHostBackup->userGet(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cmd'} = $oHostBackup->backrestExe(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-config'} = $oHostBackup->backrestConfig(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOG_PATH)} = $self->logPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_LOCK_PATH)} = $self->lockPath(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); + $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); } } @@ -1244,7 +1265,7 @@ sub manifestMunge {name => 'bCache', default => true}, ); - $self->infoMunge(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true); + $self->infoMunge($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true); # Return from function and log return values if any return logDebugReturn($strOperation); @@ -1271,7 +1292,7 @@ sub manifestRestore {name => 'bSave', default => true}, ); - $self->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $bSave); + $self->infoRestore($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $bSave); # Return from function and log return values if any return logDebugReturn($strOperation); @@ -1309,15 +1330,16 @@ sub infoMunge # If the original file content does not exist then load it if (!defined($self->{hInfoFile}{$strFileName})) { - $self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini($strFileName, {oStorage => storageRepo(), - strCipherPass => !$bManifest ? storageRepo()->cipherPassUser() : $self->cipherPassManifest()}); + $self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini( + storageRepo(), $strFileName, + {strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); } # Make a copy of the original file contents my $oMungeIni = new pgBackRest::Common::Ini( - $strFileName, - {bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), oStorage => storageRepo(), - strCipherPass => !$bManifest ? storageRepo()->cipherPassUser() : $self->cipherPassManifest()}); + storageRepo(), $strFileName, + {bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), + strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); # Load params foreach my $strSection (keys(%{$hParam})) @@ -1438,7 +1460,7 @@ sub configRecovery if (@stryRecoveryOption) { - $oConfig->{$strStanza}{cfgOptionName(CFGOPT_RECOVERY_OPTION)} = \@stryRecoveryOption; + $oConfig->{$strStanza}{'recovery-option'} = \@stryRecoveryOption; } # Save db config file @@ -1472,7 +1494,7 @@ sub configRemap } # Rewrite recovery section - delete($oConfig->{"${strStanza}:restore"}{cfgOptionName(CFGOPT_TABLESPACE_MAP)}); + delete($oConfig->{"${strStanza}:restore"}{'tablespace-map'}); my @stryTablespaceMap; foreach my $strRemap (sort(keys(%$oRemapHashRef))) @@ -1481,13 +1503,13 @@ sub configRemap if ($strRemap eq MANIFEST_TARGET_PGDATA) { - $oConfig->{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $strRemapPath; + $oConfig->{$strStanza}{'pg1-path'} = $strRemapPath; ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; if (defined($oHostBackup)) { - $oRemoteConfig->{$strStanza}{cfgOptionName(CFGOPT_PG_PATH)} = $strRemapPath; + $oRemoteConfig->{$strStanza}{'pg1-path'} = $strRemapPath; } } else @@ -1502,7 +1524,7 @@ sub configRemap if (@stryTablespaceMap) { - $oConfig->{"${strStanza}:restore"}{cfgOptionName(CFGOPT_TABLESPACE_MAP)} = \@stryTablespaceMap; + $oConfig->{"${strStanza}:restore"}{'tablespace-map'} = \@stryTablespaceMap; } # Save db config file @@ -1577,14 +1599,13 @@ sub restore $strComment = 'restore' . ($bDelta ? ' delta' : '') . ($bForce ? ', force' : '') . - ($strBackup ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? ", backup '${strBackup}'" : '') . + ($strBackup ne 'latest' ? ", backup '${strBackup}'" : '') . # This does not output 'default' for synthetic tests to make expect logs match up (may change later) ($strType ? ", type '${strType}'" : (defined($rhExpectedManifest) ? '' : ", type 'default'")) . ($strTarget ? ", target '${strTarget}'" : '') . ($strTargetTimeline ? ", timeline '${strTargetTimeline}'" : '') . ($bTargetExclusive ? ', exclusive' : '') . - (defined($strTargetAction) && $strTargetAction ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_TARGET_ACTION) - ? ', ' . cfgOptionName(CFGOPT_TARGET_ACTION) . "=${strTargetAction}" : '') . + (defined($strTargetAction) && $strTargetAction ne 'pause' ? ", target-action=${strTargetAction}" : '') . (defined($rhRemapHash) ? ', remap' : '') . (defined($iExpectedExitStatus) ? ", expect exit ${iExpectedExitStatus}" : '') . (defined($strComment) ? " - ${strComment}" : '') . @@ -1602,7 +1623,7 @@ sub restore # - which should be the backup passed as strBackupExpected. If it is not defined, then set it based on the strBackup passed. if (!defined($strBackupExpected)) { - $strBackupExpected = $strBackup eq cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? $oHostBackup->backupLast() : + $strBackupExpected = $strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup; } @@ -1610,9 +1631,7 @@ sub restore { # Load the manifest from the backup expected to be chosen/processed by restore my $oExpectedManifest = new pgBackRest::Manifest( - storageRepo()->pathGet( - STORAGE_REPO_BACKUP . qw{/} . $strBackupExpected. qw{/} . - FILE_MANIFEST), + $self->repoBackupPath($strBackupExpected . qw{/} . FILE_MANIFEST), {strCipherPass => $oHostBackup->cipherPassManifest()}); $rhExpectedManifest = $oExpectedManifest->{oContent}; @@ -1672,7 +1691,7 @@ sub restore ' --config=' . $self->backrestConfig() . ($bDelta ? ' --delta' : '') . ($bForce ? ' --force' : '') . - ($strBackup ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET) ? " --set=${strBackup}" : '') . + ($strBackup ne 'latest' ? " --set=${strBackup}" : '') . (defined($strOptionalParam) ? " ${strOptionalParam} " : '') . (defined($strType) && $strType ne CFGOPTVAL_RESTORE_TYPE_DEFAULT ? " --type=${strType}" : '') . (defined($strTarget) ? " --target=\"${strTarget}\"" : '') . @@ -1680,8 +1699,7 @@ sub restore ($bTargetExclusive ? ' --target-exclusive' : '') . (defined($strLinkMap) ? $strLinkMap : '') . ($self->synthetic() ? '' : ' --link-all') . - (defined($strTargetAction) && $strTargetAction ne cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_TARGET_ACTION) - ? ' --' . cfgOptionName(CFGOPT_TARGET_ACTION) . "=${strTargetAction}" : '') . + (defined($strTargetAction) && $strTargetAction ne 'pause' ? " --target-action=${strTargetAction}" : '') . ' --stanza=' . $self->stanza() . ' restore', {strComment => $strComment, iExpectedExitStatus => $iExpectedExitStatus, oLogTest => $self->{oLogTest}, bLogOutput => $self->synthetic()}, @@ -1723,9 +1741,8 @@ sub restoreCompare { my $oExpectedManifest = new pgBackRest::Manifest( - storageRepo()->pathGet( - STORAGE_REPO_BACKUP . qw{/} . ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . - '/'. FILE_MANIFEST), + $self->repoBackupPath( + ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST), {strCipherPass => $oHostBackup->cipherPassManifest()}); # Get the --delta option from the backup manifest so the actual manifest can be built the same way for comparison @@ -1734,9 +1751,8 @@ sub restoreCompare $oLastManifest = new pgBackRest::Manifest( - storageRepo()->pathGet( - STORAGE_REPO_BACKUP . qw{/} . - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST), + $self->repoBackupPath( + ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST), {strCipherPass => $oHostBackup->cipherPassManifest()}); } @@ -2027,6 +2043,20 @@ sub restoreCompare storageTest()->remove("${strTestPath}/actual.manifest"); } +#################################################################################################################################### +# Get repo backup/archive path +#################################################################################################################################### +sub repoSubPath +{ + my $self = shift; + my $strSubPath = shift; + my $strPath = shift; + + return + ($self->{strRepoPath} eq '/' ? '' : $self->{strRepoPath}) . "/${strSubPath}/" . $self->stanza() . + (defined($strPath) ? "/${strPath}" : ''); +} + #################################################################################################################################### # Getters #################################################################################################################################### @@ -2036,13 +2066,15 @@ sub backrestExe {return testRunGet()->backrestExe()} sub bogusHost {return shift->{bBogusHost}} sub hardLink {return shift->{bHardLink}} sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)} -sub isFS {storageRepo()->type() ne STORAGE_S3} +sub isFS {storageRepo()->type() ne STORAGE_OBJECT} sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()} sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER} sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY} sub isHostDb {my $self = shift; return $self->isHostDbMaster() || $self->isHostDbStandby()} sub lockPath {return shift->{strLockPath}} sub logPath {return shift->{strLogPath}} +sub repoArchivePath {return shift->repoSubPath('archive', shift)} +sub repoBackupPath {return shift->repoSubPath('backup', shift)} sub repoPath {return shift->{strRepoPath}} sub repoEncrypt {return shift->{bRepoEncrypt}} sub stanza {return testRunGet()->stanza()} diff --git a/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm index e77e39835..640b41a45 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm @@ -17,7 +17,7 @@ use Exporter qw(import); use File::Basename qw(dirname); use pgBackRest::Common::Log; -use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Storage::Helper; use pgBackRest::Version; use pgBackRestTest::Common::ContainerTest; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm index 8b119b704..9e2315080 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm @@ -21,10 +21,9 @@ use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::String; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::DbVersion; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Storage::Helper; use pgBackRest::Version; use pgBackRestTest::Env::Host::HostBackupTest; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm index 19479e7bd..329e2a96f 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm @@ -23,7 +23,7 @@ use pgBackRest::Common::String; use pgBackRest::Common::Wait; use pgBackRest::DbVersion; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Storage::Helper; use pgBackRest::Version; use pgBackRestTest::Env::Host::HostBackupTest; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm index 2cd82f690..38439ed51 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm @@ -22,7 +22,7 @@ use pgBackRest::Common::String; use pgBackRest::Common::Wait; use pgBackRest::DbVersion; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Storage::Helper; use pgBackRest::Version; use pgBackRestTest::Env::Host::HostBackupTest; diff --git a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm index a5e1a04d3..f6a8b4bf6 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm @@ -22,9 +22,8 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Storage::Helper; use pgBackRest::Version; use pgBackRestTest::Env::Host::HostBaseTest; diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm index e13a287ee..fe4826ced 100644 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ b/test/lib/pgBackRestTest/Env/HostEnvTest.pm @@ -2,7 +2,7 @@ # FullCommonTest.pm - Common code for backup tests #################################################################################################################################### package pgBackRestTest::Env::HostEnvTest; -use parent 'pgBackRestTest::Env::ConfigEnvTest'; +use parent 'pgBackRestTest::Common::RunTest'; #################################################################################################################################### # Perl includes @@ -18,9 +18,9 @@ use Storable qw(dclone); use pgBackRest::Archive::Common; use pgBackRest::Common::Log; -use pgBackRest::Config::Config; use pgBackRest::DbVersion; -use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Storage::Base; +use pgBackRest::Storage::Helper; use pgBackRestTest::Env::Host::HostBackupTest; use pgBackRestTest::Env::Host::HostBaseTest; @@ -43,11 +43,6 @@ use constant ENCRYPTION_KEY_MANIFEST => 'manifest'; use constant ENCRYPTION_KEY_BACKUPSET => 'backupset'; push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET); -use constant NONE => CFGOPTVAL_COMPRESS_TYPE_NONE; - push @EXPORT, qw(NONE); -use constant GZ => CFGOPTVAL_COMPRESS_TYPE_GZ; - push @EXPORT, qw(GZ); - #################################################################################################################################### # setup #################################################################################################################################### @@ -151,6 +146,13 @@ sub setup $oHostBackup = $strBackupDestination eq HOST_DB_MASTER ? $oHostDbMaster : $oHostDbStandby; } + storageRepoCommandSet( + $self->backrestExeHelper() . + ' --config=' . $oHostBackup->backrestConfig() . ' --stanza=' . $self->stanza() . ' --log-level-console=off' . + ' --log-level-stderr=error' . + ($oConfigParam->{bS3} ? ' --no-repo1-s3-verify-tls --repo1-s3-host=' . $oHostS3->ipGet() : ''), + $oConfigParam->{bS3} ? STORAGE_OBJECT : STORAGE_POSIX); + # Create db-standby config if (defined($oHostDbStandby)) { @@ -161,37 +163,10 @@ sub setup bArchiveAsync => $$oConfigParam{bArchiveAsync}}); } - # Set options needed for storage helper - $self->optionTestSet(CFGOPT_PG_PATH, $oHostDbMaster->dbBasePath()); - $self->optionTestSet(CFGOPT_REPO_PATH, $oHostBackup->repoPath()); - $self->optionTestSet(CFGOPT_STANZA, $self->stanza()); - - # Configure the repo to be encrypted if required - if ($bRepoEncrypt) - { - $self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC); - $self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, 'x'); - } - - # Set S3 options - if (defined($oHostS3)) - { - $self->optionTestSet(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3); - $self->optionTestSet(CFGOPT_REPO_S3_KEY, HOST_S3_ACCESS_KEY); - $self->optionTestSet(CFGOPT_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY); - $self->optionTestSet(CFGOPT_REPO_S3_BUCKET, HOST_S3_BUCKET); - $self->optionTestSet(CFGOPT_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT); - $self->optionTestSet(CFGOPT_REPO_S3_REGION, HOST_S3_REGION); - $self->optionTestSet(CFGOPT_REPO_S3_HOST, $oHostS3->ipGet()); - $self->optionTestSetBool(CFGOPT_REPO_S3_VERIFY_TLS, false); - } - - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - # Create S3 bucket if (defined($oHostS3)) { - storageRepo()->{oStorageC}->bucketCreate(); + storageRepo()->create(); } return $oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3; diff --git a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm index 41010c4c3..0624598b0 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm @@ -21,10 +21,8 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::InfoCommon; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; use pgBackRest::Version; @@ -93,12 +91,12 @@ sub run # without slowing down the other tests too much. if ($bS3) { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}}); - $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}}); + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); + $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); # Reduce log level to warn because parallel tests do not create deterministic logs - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(WARN)}}); - $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(WARN)}}); + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(WARN)}}); + $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(WARN)}}); } # Get base time @@ -115,7 +113,7 @@ sub run $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 16384; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE} = JSON::PP::true; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::false; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = CFGOPTVAL_COMPRESS_TYPE_NONE; + $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = NONE; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL} = 3; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK} = $bRemote ? 1 : 3; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::false; @@ -306,7 +304,7 @@ sub run my $strTestPoint; # Create the archive info file - $oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + $oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-online'}); # Create a link to postgresql.conf storageTest()->pathCreate($oHostDbMaster->dbPath() . '/pg_config', {strMode => '0700', bCreateParent => true}); @@ -343,14 +341,13 @@ sub run {oExpectedManifest => \%oManifest, strOptionalParam => $strOptionalParam . # Pass ssh path to make sure it is used - ($bRemote ? ' --' . cfgOptionName(CFGOPT_CMD_SSH) . '=/usr/bin/ssh' : '') . + ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . # Pass bogus ssh port to make sure it is passed through the protocol layer (it won't be used) - ($bRemote ? ' --' . cfgOptionName(CFGOPT_PG_PORT) . '=9999' : '') . + ($bRemote ? ' --pg1-port=9999' : '') . # Pass bogus socket path to make sure it is passed through the protocol layer (it won't be used) - ($bRemote ? ' --' . cfgOptionName(CFGOPT_PG_SOCKET_PATH) . '=/test_socket_path' : '') . - ' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=16384 --' . cfgOptionName(CFGOPT_CHECKSUM_PAGE) . - ' --' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1', - strRepoType => $bS3 ? undef : CFGOPTVAL_REPO_TYPE_CIFS, strTest => $strTestPoint, fTestDelay => 0}); + ($bRemote ? ' --pg1-socket-path=/test_socket_path' : '') . + ' --buffer-size=16384 --checksum-page --process-max=1', + strRepoType => $bS3 ? undef : STORAGE_CIFS, strTest => $strTestPoint, fTestDelay => 0}); $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 65536; @@ -409,13 +406,13 @@ sub run # Resume by copying the valid full backup over the last aborted full backup if it exists, or by creating a new path my $strResumeBackup = (storageRepo()->list( - STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))[0]; + $oHostBackup->repoBackupPath(), {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))[0]; my $strResumeLabel = $strResumeBackup ne $strFullBackup ? - $strResumeBackup : backupLabel(storageRepo(), $strType, undef, time()); - my $strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . '/' . $strResumeLabel); + $strResumeBackup : backupLabel(storageRepo(), $oHostBackup->repoBackupPath(), $strType, undef, time()); + my $strResumePath = $oHostBackup->repoBackupPath($strResumeLabel); forceStorageRemove(storageRepo(), $strResumePath, {bRecurse => true}); - forceStorageMove(storageRepo(), 'backup/' . $self->stanza() . "/${strFullBackup}", $strResumePath); + forceStorageMove(storageRepo(), $oHostBackup->repoBackupPath($strFullBackup), $strResumePath); # Set ownership on base directory to bogus values if (!$bRemote) @@ -464,7 +461,7 @@ sub run # Create files to be excluded with the --exclude option $oHostBackup->configUpdate( {(CFGDEF_SECTION_GLOBAL . ':backup') => - {cfgOptionName(CFGOPT_EXCLUDE) => ['postgresql.auto.conf', 'pg_log/', 'pg_log2', 'apipe']}}); + {'exclude' => ['postgresql.auto.conf', 'pg_log/', 'pg_log2', 'apipe']}}); $oHostDbMaster->dbLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.auto.conf', '../pg_config/postgresql.conf', true); $oHostDbMaster->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_log'); @@ -476,7 +473,7 @@ sub run $strFullBackup = $oHostBackup->backup( $strType, 'resume', {oExpectedManifest => \%oManifest, - strOptionalParam => '--force --' . cfgOptionName(CFGOPT_CHECKSUM_PAGE) . ($bDeltaBackup ? ' --delta' : '')}); + strOptionalParam => '--force --checksum-page' . ($bDeltaBackup ? ' --delta' : '')}); # Remove postmaster.pid so restore will succeed (the rest will be cleaned up by the delta) storageTest->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID); @@ -523,7 +520,7 @@ sub run strOptionalParam => ' --link-all' . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '')}); # Remove excludes now that they just create noise in the log - $oHostBackup->configUpdate({(CFGDEF_SECTION_GLOBAL . ':backup') => {cfgOptionName(CFGOPT_EXCLUDE) => []}}); + $oHostBackup->configUpdate({(CFGDEF_SECTION_GLOBAL . ':backup') => {'exclude' => []}}); # Run again to fix permissions if (!$bRemote) @@ -624,11 +621,11 @@ sub run $strType = CFGOPTVAL_BACKUP_TYPE_INCR; # Create resumable backup from last backup - $strResumeLabel = backupLabel(storageRepo(), $strType, substr($strBackup, 0, 16), time()); - $strResumePath = storageRepo()->pathGet('backup/' . $self->stanza() . '/' . $strResumeLabel); + $strResumeLabel = backupLabel(storageRepo(), $oHostBackup->repoBackupPath(), $strType, substr($strBackup, 0, 16), time()); + $strResumePath = $oHostBackup->repoBackupPath($strResumeLabel); forceStorageRemove(storageRepo(), $strResumePath); - forceStorageMove(storageRepo(), 'backup/' . $self->stanza() . "/${strBackup}", $strResumePath); + forceStorageMove(storageRepo(), $oHostBackup->repoBackupPath($strBackup), $strResumePath); # Munge manifest so the resumed file in the repo appears to be bad if ($bEncrypt || $bRemote) @@ -698,7 +695,7 @@ sub run $strBackup = $oHostBackup->backup( $strType, 'resume and add tablespace 2', {oExpectedManifest => \%oManifest, - strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')}); + strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')}); $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1; @@ -719,9 +716,7 @@ sub run $strBackup = $oHostBackup->backup( $strType, 'drop tablespace 11', - {oExpectedManifest => \%oManifest, - strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . - ($bDeltaBackup ? ' --delta' : '')}); + {oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')}); $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1; @@ -776,7 +771,7 @@ sub run $strBackup = $oHostBackup->backup( $strType, 'add files and remove tablespace 2', - {oExpectedManifest => \%oManifest, strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1'}); + {oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1'}); $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1; @@ -801,7 +796,7 @@ sub run $strBackup = $oHostBackup->backup( $strType, 'updates since last full', {oExpectedManifest => \%oManifest, - strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')}); + strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')}); $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1; @@ -814,17 +809,17 @@ sub run # Enable compression to ensure a warning is raised (reset when gz to avoid log churn since it is the default) if ($strCompressType eq GZ) { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => undef}}); + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'compress-type' => undef}}); } else { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => $strCompressType}}); + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'compress-type' => $strCompressType}}); } # Enable hardlinks (except for s3) to ensure a warning is raised if (!$bS3) { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_REPO_HARDLINK) => 'y'}}); + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'repo1-hardlink' => 'y'}}); } $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1; @@ -840,7 +835,7 @@ sub run $oHostBackup->backup( $strType, 'remove files', {oExpectedManifest => \%oManifest, - strOptionalParam => '--' . cfgOptionName(CFGOPT_PROCESS_MAX) . '=1' . ($bDeltaBackup ? ' --delta' : '')}); + strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')}); $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $bS3 ? 2 : 1; @@ -889,8 +884,7 @@ sub run $strFullBackup, {&MANIFEST_SECTION_BACKUP_OPTION => {&MANIFEST_KEY_CHECKSUM_PAGE => undef}}, false); $strBackup = $oHostBackup->backup( - $strType, 'add file', - {oExpectedManifest => \%oManifest, strOptionalParam => '--' . cfgOptionName(CFGOPT_CHECKSUM_PAGE)}); + $strType, 'add file', {oExpectedManifest => \%oManifest, strOptionalParam => '--checksum-page'}); # Selective Restore #--------------------------------------------------------------------------------------------------------------------------- @@ -906,7 +900,7 @@ sub run {&MANIFEST_SUBKEY_CHECKSUM}); $oHostDbMaster->restore( - 'selective restore 16384', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + 'selective restore 16384', 'latest', {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true, strOptionalParam => '--db-include=16384'}); @@ -924,7 +918,7 @@ sub run delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/16384/17000'}{&MANIFEST_SUBKEY_CHECKSUM}); $oHostDbMaster->restore( - 'selective restore 32768', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + 'selective restore 32768', 'latest', {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true, strOptionalParam => '--db-include=32768'}); @@ -932,12 +926,12 @@ sub run '7579ada0808d7f98087a0a586d0df9de009cdc33'; $oHostDbMaster->restore( - 'error on invalid id', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + 'error on invalid id', 'latest', {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true, iExpectedExitStatus => ERROR_DB_MISSING, strOptionalParam => '--log-level-console=warn --db-include=7777'}); $oHostDbMaster->restore( - 'error on system id', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + 'error on system id', 'latest', {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true, iExpectedExitStatus => ERROR_DB_INVALID, strOptionalParam => '--log-level-console=warn --db-include=1'}); @@ -952,7 +946,7 @@ sub run delete($oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'}); $oHostDbMaster->restore( - 'no tablespace remap', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + 'no tablespace remap', 'latest', {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bTablespace => false, strOptionalParam => '--tablespace-map-all=../../tablespace'}); @@ -965,8 +959,9 @@ sub run #--------------------------------------------------------------------------------------------------------------------------- if (!$bRemote && !$bS3) { - executeTest('ls -1Rtr ' . storageRepo()->pathGet('backup/' . $self->stanza() . '/' . PATH_BACKUP_HISTORY), - {oLogTest => $self->expect(), bRemote => $bRemote}); + executeTest( + 'ls -1Rtr ' . $oHostBackup->repoBackupPath(PATH_BACKUP_HISTORY), + {oLogTest => $self->expect(), bRemote => $bRemote}); } # Test backup from standby warning that standby not configured so option reset @@ -975,7 +970,7 @@ sub run { $strBackup = $oHostBackup->backup( $strType, 'option backup-standby reset - backup performed from master', {oExpectedManifest => \%oManifest, - strOptionalParam => '--log-level-console=info --' . cfgOptionName(CFGOPT_BACKUP_STANDBY)}); + strOptionalParam => '--log-level-console=info --backup-standby'}); } } } diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm index 9df346a0f..547ebd9ed 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm @@ -20,12 +20,11 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; use pgBackRestTest::Env::HostEnvTest; +use pgBackRestTest::Env::Host::HostBackupTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::RunTest; use pgBackRestTest::Common::VmTest; @@ -86,7 +85,7 @@ sub run $self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94); # Create the archive info file - $oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + $oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-online'}); # Push a WAL segment &log(INFO, ' push first WAL'); @@ -96,7 +95,7 @@ sub run if ($iError == 0) { $oHostBackup->infoMunge( - storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), + $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE), {&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}, &INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}}); } @@ -120,13 +119,12 @@ sub run # Fix the database version if ($iError == 0) { - $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)); + $oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE)); } #--------------------------------------------------------------------------------------------------------------------------- $self->testResult( - sub {storageRepo()->list( - STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')}, + sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))}, "000000010000000100000001-${strWalHash}${strCompressExt}", 'segment 2-4 not pushed', {iWaitSeconds => 5}); @@ -134,8 +132,7 @@ sub run $oHostDbMaster->archivePush($strWalPath, $strWalTestFile, 5); $self->testResult( - sub {storageRepo()->list( - STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')}, + sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))}, "(000000010000000100000001-${strWalHash}${strCompressExt}, " . "000000010000000100000005-${strWalHash}${strCompressExt})", 'segment 5 is pushed', {iWaitSeconds => 5}); diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm index 3c6d880ff..bed83a928 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm @@ -20,11 +20,10 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; +use pgBackRestTest::Env::Host::HostBackupTest; use pgBackRestTest::Env::HostEnvTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::RunTest; @@ -38,13 +37,15 @@ use pgBackRestTest::Common::VmTest; sub archiveCheck { my $self = shift; + my $oHostBackup = shift; my $strArchiveFile = shift; my $strArchiveChecksum = shift; my $strCompressType = shift; my $strSpoolPath = shift; # Build the archive name to check for at the destination - my $strArchiveCheck = PG_VERSION_94 . "-1/${strArchiveFile}-${strArchiveChecksum}"; + my $strArchiveCheck = $oHostBackup->repoArchivePath( + PG_VERSION_94 . "-1/" . substr($strArchiveFile, 0, 16) . "/${strArchiveFile}-${strArchiveChecksum}"); if (defined($strCompressType)) { @@ -56,13 +57,13 @@ sub archiveCheck do { - $bFound = storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}"); + $bFound = storageRepo()->exists($strArchiveCheck); } while (!$bFound && waitMore($oWait)); if (!$bFound) { - confess 'unable to find ' . storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveCheck}"); + confess "unable to find ${strArchiveCheck}"; } if (defined($strSpoolPath)) @@ -108,7 +109,7 @@ sub run true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE}); # Reduce console logging to detail - $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(DETAIL)}}); + $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(DETAIL)}}); # Create the wal path my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog'; @@ -121,11 +122,11 @@ sub run # Create archive-push command my $strCommandPush = $oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=' . $self->stanza() . - ' ' . cfgCommandName(CFGCMD_ARCHIVE_PUSH); + ' archive-push'; my $strCommandGet = $oHostDbMaster->backrestExe() . ' --config=' . $oHostDbMaster->backrestConfig() . ' --stanza=' . $self->stanza() . - ' ' . cfgCommandName(CFGCMD_ARCHIVE_GET); + ' archive-get'; #--------------------------------------------------------------------------------------------------------------------------- &log(INFO, ' archive.info missing'); @@ -142,9 +143,7 @@ sub run {iExpectedExitStatus => ERROR_FILE_MISSING, oLogTest => $self->expect()}); #--------------------------------------------------------------------------------------------------------------------------- - $oHostBackup->stanzaCreate( - 'stanza create', - {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + $oHostBackup->stanzaCreate('stanza create', {strOptionalParam => '--no-online'}); #--------------------------------------------------------------------------------------------------------------------------- &log(INFO, ' push first WAL'); @@ -160,7 +159,7 @@ sub run push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}"); # Test that the WAL was pushed - $self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType); + $self->archiveCheck($oHostBackup, $strSourceFile, $strArchiveChecksum, $strCompressType); # Remove from archive_status storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready"); @@ -245,7 +244,7 @@ sub run } # Test that the WAL was pushed - $self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbMaster->spoolPath()); + $self->archiveCheck($oHostBackup, $strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbMaster->spoolPath()); # Remove from archive_status storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready"); @@ -263,7 +262,7 @@ sub run "${strCommandPush} --archive-async ${strWalPath}/00000002.history", {oLogTest => $self->expect()}); - if (!storageRepo()->exists(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/00000002.history')) + if (!storageRepo()->exists($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/00000002.history'))) { confess 'unable to find history file in archive'; } @@ -275,7 +274,7 @@ sub run # db section and corresponding history munged $oHostBackup->infoMunge( - storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), + $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE), {&INFO_ARCHIVE_SECTION_DB_HISTORY => {'1' => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}}); $oHostDbMaster->executeSimple( @@ -283,13 +282,13 @@ sub run {iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()}); # Restore the file to its original condition - $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)); + $oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE)); #--------------------------------------------------------------------------------------------------------------------------- &log(INFO, ' db system-id mismatch error'); $oHostBackup->infoMunge( - storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), + $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE), {&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}, &INFO_ARCHIVE_SECTION_DB_HISTORY => {'1' => {&INFO_ARCHIVE_KEY_DB_ID => 5000900090001855000}}}); @@ -302,7 +301,7 @@ sub run {iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()}); # Restore the file to its original condition - $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)); + $oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE)); #--------------------------------------------------------------------------------------------------------------------------- &log(INFO, ' stop'); @@ -384,7 +383,7 @@ sub run $oHostDbMaster->executeSimple( $strCommandPush . " ${strWalPath}/${strSourceFile}.partial", {oLogTest => $self->expect()}); - $self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum); + $self->archiveCheck($oHostBackup, "${strSourceFile}.partial", $strArchiveChecksum); push(@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}"); @@ -393,7 +392,7 @@ sub run $oHostDbMaster->executeSimple( $strCommandPush . " ${strWalPath}/${strSourceFile}.partial", {oLogTest => $self->expect()}); - $self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum); + $self->archiveCheck($oHostBackup, "${strSourceFile}.partial", $strArchiveChecksum); #--------------------------------------------------------------------------------------------------------------------------- &log(INFO, ' .partial WAL with different checksum'); @@ -405,7 +404,7 @@ sub run #--------------------------------------------------------------------------------------------------------------------------- $self->testResult( - sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/0000000100000001')}, + sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))}, '(' . join(', ', @stryExpectedWAL) . ')', 'all WAL in archive', {iWaitSeconds => 5}); } diff --git a/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm index e5831b60f..ec7d2bf3c 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm @@ -21,50 +21,17 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Storage::Helper; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::RunTest; use pgBackRestTest::Common::VmTest; use pgBackRestTest::Env::ExpireEnvTest; +use pgBackRestTest::Env::Host::HostBackupTest; use pgBackRestTest::Env::Host::HostS3Test; use pgBackRestTest::Env::HostEnvTest; -#################################################################################################################################### -# initStanzaOption -#################################################################################################################################### -sub initStanzaOption -{ - my $self = shift; - my $strDbBasePath = shift; - my $strRepoPath = shift; - my $oHostS3 = shift; - - $self->optionTestSet(CFGOPT_STANZA, $self->stanza()); - $self->optionTestSet(CFGOPT_PG_PATH, $strDbBasePath); - $self->optionTestSet(CFGOPT_REPO_PATH, $strRepoPath); - $self->optionTestSet(CFGOPT_LOG_PATH, $self->testPath()); - - $self->optionTestSetBool(CFGOPT_ONLINE, false); - - $self->optionTestSet(CFGOPT_DB_TIMEOUT, 5); - $self->optionTestSet(CFGOPT_PROTOCOL_TIMEOUT, 6); - - if (defined($oHostS3)) - { - $self->optionTestSet(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3); - $self->optionTestSet(CFGOPT_REPO_S3_KEY, HOST_S3_ACCESS_KEY); - $self->optionTestSet(CFGOPT_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY); - $self->optionTestSet(CFGOPT_REPO_S3_BUCKET, HOST_S3_BUCKET); - $self->optionTestSet(CFGOPT_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT); - $self->optionTestSet(CFGOPT_REPO_S3_REGION, HOST_S3_REGION); - $self->optionTestSet(CFGOPT_REPO_S3_HOST, $oHostS3->ipGet()); - $self->optionTestSetBool(CFGOPT_REPO_S3_VERIFY_TLS, false); - } -} - #################################################################################################################################### # run #################################################################################################################################### @@ -103,12 +70,9 @@ sub run my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( true, $self->expect(), {bS3 => $bS3, bRepoEncrypt => $bEncrypt}); - $self->initStanzaOption($oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3); - $self->configTestLoad(CFGCMD_STANZA_CREATE); - # Create the test object my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest( - $oHostBackup, $self->backrestExe(), storageRepo(), $self->expect(), $self); + $oHostBackup, $self->backrestExe(), storageRepo(), $oHostDbMaster->dbPath(), $self->expect(), $self); $oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_92); @@ -202,12 +166,9 @@ sub run my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( true, $self->expect(), {bS3 => $bS3, bRepoEncrypt => $bEncrypt}); - $self->initStanzaOption($oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3); - $self->configTestLoad(CFGCMD_STANZA_CREATE); - # Create the test object my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest( - $oHostBackup, $self->backrestExe(), storageRepo(), $self->expect(), $self); + $oHostBackup, $self->backrestExe(), storageRepo(), $oHostDbMaster->dbPath(), $self->expect(), $self); $oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_92); diff --git a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm index 8405a7d70..8c616b6ee 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm @@ -19,14 +19,13 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::DbVersion; use pgBackRest::InfoCommon; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Base; use pgBackRest::Storage::Helper; +use pgBackRestTest::Env::Host::HostBackupTest; use pgBackRestTest::Env::HostEnvTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::FileTest; @@ -40,17 +39,6 @@ sub run { my $self = shift; - # Archive and backup info file names - my $strArchiveInfoFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE; - my $strArchiveInfoCopyFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT; - my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old"; - my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old"; - - my $strBackupInfoFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO; - my $strBackupInfoCopyFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT; - my $strBackupInfoOldFile = "${strBackupInfoFile}.old"; - my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old"; - foreach my $rhRun ( {vm => VM1, remote => false, s3 => false, encrypt => true, compress => GZ}, @@ -80,9 +68,20 @@ sub run true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => $strCompressType}); + # Archive and backup info file names + my $strArchiveInfoFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE); + my $strArchiveInfoCopyFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE . INI_COPY_EXT); + my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old"; + my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old"; + + my $strBackupInfoFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO); + my $strBackupInfoCopyFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO . INI_COPY_EXT); + my $strBackupInfoOldFile = "${strBackupInfoFile}.old"; + my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old"; + # Create the stanza $oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_MISSING, - strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_LOG_LEVEL_FILE) . '=info'}); + strOptionalParam => '--no-online --log-level-file=info'}); # Generate pg_control for stanza-create storageTest()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true}); @@ -91,17 +90,17 @@ sub run # Fail stanza upgrade before stanza-create has been performed #-------------------------------------------------------------------------------------------------------------------------- $oHostBackup->stanzaUpgrade('fail on stanza not initialized since archive.info is missing', - {iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + {iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-online'}); # Create the stanza successfully without force #-------------------------------------------------------------------------------------------------------------------------- - $oHostBackup->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + $oHostBackup->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-online'}); # Rerun stanza-create and confirm it does not fail #-------------------------------------------------------------------------------------------------------------------------- $oHostBackup->stanzaCreate( 'do not fail on rerun of stanza-create - info files exist and DB section ok', - {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + {strOptionalParam => '--no-online'}); # Stanza Create fails when not using force - database mismatch with pg_control file #-------------------------------------------------------------------------------------------------------------------------- @@ -109,15 +108,14 @@ sub run $self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94); $oHostBackup->stanzaCreate('fail on database mismatch and warn force option deprecated', - {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . - ' --' . cfgOptionName(CFGOPT_FORCE)}); + {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-online --force'}); # Restore pg_control $self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_93); # Perform a stanza upgrade which will indicate already up to date #-------------------------------------------------------------------------------------------------------------------------- - $oHostBackup->stanzaUpgrade('already up to date', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + $oHostBackup->stanzaUpgrade('already up to date', {strOptionalParam => '--no-online'}); # Create the wal path my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog'; @@ -140,7 +138,7 @@ sub run if (!$bEncrypt) { $oHostBackup->stanzaCreate('fail on archive info file missing from non-empty dir', - {iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + {iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-online'}); } # Restore info files from copy @@ -170,8 +168,7 @@ sub run # Save a pre-upgrade copy of archive info fo testing db-id mismatch forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false}); - $oHostBackup->stanzaUpgrade('successful upgrade creates additional history', {strOptionalParam => '--no-' . - cfgOptionName(CFGOPT_ONLINE)}); + $oHostBackup->stanzaUpgrade('successful upgrade creates additional history', {strOptionalParam => '--no-online'}); # Make sure that WAL from the old version can still be retrieved #-------------------------------------------------------------------------------------------------------------------------- @@ -193,7 +190,7 @@ sub run # Push a WAL segment so have a valid file in the latest DB archive dir only $oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1); $self->testResult( - sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-2/0000000100000001')}, + sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-2/0000000100000001'))}, '000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_94) . ".${strCompressType}", 'check that WAL is in the archive at -2'); @@ -201,8 +198,7 @@ sub run #-------------------------------------------------------------------------------------------------------------------------- storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_PGTBLSPC); $oHostBackup->backup( - 'full', 'create first full backup ', - {strOptionalParam => '--repo1-retention-full=2 --no-' . cfgOptionName(CFGOPT_ONLINE)}, false); + 'full', 'create first full backup ', {strOptionalParam => '--repo1-retention-full=2 --no-online'}, false); # Upgrade the stanza #-------------------------------------------------------------------------------------------------------------------------- @@ -210,8 +206,7 @@ sub run $self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_95); forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600'); - - $oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + $oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-online'}); # Copy archive.info and restore really old version forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false}); @@ -219,16 +214,15 @@ sub run forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoFile, {bRecurse => false}); # Confirm versions - my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza())); - my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza())); + my $oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath()); + my $oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath()); $self->testResult(sub {$oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, PG_VERSION_93)}, true, 'archive at old pg version'); $self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, PG_VERSION_95)}, true, 'backup at new pg version'); $oHostBackup->stanzaUpgrade( - 'upgrade fails with mismatched db-ids', - {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + 'upgrade fails with mismatched db-ids', {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-online'}); # Restore archive.info forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false}); @@ -239,8 +233,8 @@ sub run $oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1); # Test backup is changed from type=DIFF to FULL (WARN message displayed) - my $oExecuteBackup = $oHostBackup->backupBegin('diff', 'diff changed to full backup', - {strOptionalParam => '--repo1-retention-full=2 --no-' . cfgOptionName(CFGOPT_ONLINE)}); + my $oExecuteBackup = $oHostBackup->backupBegin( + 'diff', 'diff changed to full backup', {strOptionalParam => '--repo1-retention-full=2 --no-online'}); $oHostBackup->backupEnd('full', $oExecuteBackup, undef, false); # Delete the stanza diff --git a/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm b/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm index 50547cf4b..5fc759114 100644 --- a/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm @@ -16,7 +16,6 @@ use Storable qw(dclone); use Time::HiRes qw(gettimeofday); use pgBackRest::Common::Log; -use pgBackRest::Config::Config; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::RunTest; @@ -63,11 +62,8 @@ sub run for (my $iIndex = 0; $iIndex < $iRunTotal; $iIndex++) { executeTest( - $self->backrestExe() . ' --' . cfgOptionName(CFGOPT_STANZA) . '=' . $self->stanza() . - ' --' . cfgOptionName(CFGOPT_ARCHIVE_ASYNC) . - ' --' . cfgOptionName(CFGOPT_SPOOL_PATH) . '=' . $self->{strSpoolPath} . - ' --' . cfgOptionName(CFGOPT_ARCHIVE_TIMEOUT) . '=1' . - ' archive-push /pg_xlog/000000010000000100000001'); + $self->backrestExe() . ' --stanza=' . $self->stanza() . ' --archive-async --spool-path=' . $self->{strSpoolPath} . + ' --archive-timeout=1 archive-push /pg_xlog/000000010000000100000001'); } &log(INFO, 'time per execution: ' . ((gettimeofday() - $lTimeBegin) / $iRunTotal)); diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm index 91a79f08e..eb8dd53e4 100644 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm @@ -20,10 +20,8 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Ini; use pgBackRest::Common::Log; use pgBackRest::Common::Wait; -use pgBackRest::Config::Config; use pgBackRest::InfoCommon; use pgBackRest::Manifest; -use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; use pgBackRest::Version; @@ -111,8 +109,8 @@ sub run # without slowing down the other tests too much. if ($bS3) { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}}); - $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_PROCESS_MAX) => 2}}); + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); + $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); } $oHostDbMaster->clusterCreate(); @@ -122,7 +120,7 @@ sub run # Get passphrase to access the Manifest file from backup.info - returns undefined if repo not encrypted my $strCipherPass = - (new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP)))->cipherPassSub(); + (new pgBackRest::Backup::Info($oHostBackup->repoBackupPath()))->cipherPassSub(); # Create a manifest with the pg version to get version-specific paths my $oManifest = new pgBackRest::Manifest(BOGUS, {bLoad => false, strDbVersion => $self->pgVersion(), @@ -157,13 +155,13 @@ sub run my $strComment = undef; # Archive and backup info file names - my $strArchiveInfoFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE; - my $strArchiveInfoCopyFile = STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT; + my $strArchiveInfoFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE); + my $strArchiveInfoCopyFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE . INI_COPY_EXT); my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old"; my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old"; - my $strBackupInfoFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO; - my $strBackupInfoCopyFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT; + my $strBackupInfoFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO); + my $strBackupInfoCopyFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO . INI_COPY_EXT); my $strBackupInfoOldFile = "${strBackupInfoFile}.old"; my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old"; @@ -232,7 +230,7 @@ sub run # load the archive info file and munge it for testing by breaking the database version $oHostBackup->infoMunge( - storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE), + $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE), {&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}, &INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}}); @@ -245,7 +243,7 @@ sub run } # Restore the file to its original condition - $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)); + $oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE)); # Check archive_timeout error when WAL segment is not found $strComment = 'fail on archive timeout'; @@ -268,7 +266,7 @@ sub run # Load the backup.info file and munge it for testing by breaking the database version and system id $oHostBackup->infoMunge( - storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO), + $oHostBackup->repoBackupPath(FILE_BACKUP_INFO), {&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_DB_VERSION => '8.0', &INFO_BACKUP_KEY_SYSTEM_ID => 6999999999999999999}, &INFO_BACKUP_SECTION_DB_HISTORY => @@ -284,7 +282,7 @@ sub run } # Restore the file to its original condition - $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)); + $oHostBackup->infoRestore($oHostBackup->repoBackupPath(FILE_BACKUP_INFO)); # ??? Removed temporarily until manifest build can be brought back into the check command # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check @@ -332,22 +330,20 @@ sub run # Run stanza-create online to confirm proper handling of configValidation error against new pg-path $oHostBackup->stanzaCreate('fail on database mismatch with directory', - {strOptionalParam => ' --' . cfgOptionName(CFGOPT_PG_PATH) . '=' . $oHostDbMaster->dbPath() . - '/testbase/', iExpectedExitStatus => ERROR_DB_MISMATCH}); + {strOptionalParam => ' --pg1-path=' . $oHostDbMaster->dbPath() . '/testbase/', + iExpectedExitStatus => ERROR_DB_MISMATCH}); # Remove the directories to be able to create the stanza - forceStorageRemove(storageRepo(), STORAGE_REPO_BACKUP, {bRecurse => true}); - forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE, {bRecurse => true}); + forceStorageRemove(storageRepo(), $oHostBackup->repoBackupPath(), {bRecurse => true}); + forceStorageRemove(storageRepo(), $oHostBackup->repoArchivePath(), {bRecurse => true}); # Stanza Upgrade - tests configValidate code - all other tests in synthetic integration tests #----------------------------------------------------------------------------------------------------------------------- # Run stanza-create offline to create files needing to be upgraded (using new pg-path) $oHostBackup->stanzaCreate('successfully create stanza files to be upgraded', - {strOptionalParam => - ' --' . cfgOptionName(CFGOPT_PG_PATH) . '=' . $oHostDbMaster->dbPath() . - '/testbase/ --no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)}); - my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza())); - my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza())); + {strOptionalParam => ' --pg1-path=' . $oHostDbMaster->dbPath() . '/testbase/ --no-online --force'}); + my $oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath()); + my $oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath()); # Read info files to confirm the files were created with a different database version if ($self->pgVersion() eq PG_VERSION_94) @@ -369,8 +365,8 @@ sub run $oHostBackup->stanzaUpgrade('upgrade stanza files online'); # Reread the info files and confirm the result - $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza())); - $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza())); + $oArchiveInfo = new pgBackRest::Archive::Info($oHostBackup->repoArchivePath()); + $oBackupInfo = new pgBackRest::Backup::Info($oHostBackup->repoBackupPath()); $self->testResult(sub {$oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $self->pgVersion())}, true, 'archive upgrade online corrects db'); $self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, @@ -409,10 +405,10 @@ sub run my $strFullBackup = $oHostBackup->backup( CFGOPTVAL_BACKUP_TYPE_FULL, 'update during backup', - {strOptionalParam => ' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=16384'}); + {strOptionalParam => ' --buffer-size=16384'}); # Enabled async archiving - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_ARCHIVE_ASYNC) => 'y'}}); + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'archive-async' => 'y'}}); # Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it # down enough to make it evident that the async process is working. @@ -444,7 +440,7 @@ sub run } $oHostDbStandby->restore( - 'restore backup on replica', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + 'restore backup on replica', 'latest', {rhRemapHash => \%oRemapHash, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, strOptionalParam => ' --recovery-option="primary_conninfo=host=' . HOST_DB_MASTER . @@ -470,19 +466,13 @@ sub run { my $strStandbyBackup = $oHostBackup->backup( CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to reach master', - {bStandby => true, - iExpectedExitStatus => ERROR_DB_CONNECT, - strOptionalParam => '--' . - cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, cfgOptionIndexTotal(CFGOPT_PG_PATH))) . '=' . BOGUS}); + {bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg8-host=' . BOGUS}); } else { my $strStandbyBackup = $oHostBackup->backup( CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to access at least one standby', - {bStandby => true, - iExpectedExitStatus => ERROR_DB_CONNECT, - strOptionalParam => '--' . - cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, cfgOptionIndexTotal(CFGOPT_PG_PATH))) . '=' . BOGUS}); + {bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg8-host=' . BOGUS}); } } @@ -490,7 +480,7 @@ sub run CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby', {bStandby => true, iExpectedExitStatus => $oHostDbStandby->pgVersion() >= PG_VERSION_BACKUP_STANDBY ? undef : ERROR_CONFIG, - strOptionalParam => '--' . cfgOptionName(CFGOPT_REPO_RETENTION_FULL) . '=1'}); + strOptionalParam => '--repo1-retention-full=1'}); if ($oHostDbStandby->pgVersion() >= PG_VERSION_BACKUP_STANDBY) { @@ -632,9 +622,7 @@ sub run # Exercise --delta checksum option my $strIncrBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'update during backup', - {strOptionalParam => - '--' . cfgOptionName(CFGOPT_STOP_AUTO) . ' --' . cfgOptionName(CFGOPT_BUFFER_SIZE) . '=32768 --delta'}); + CFGOPTVAL_BACKUP_TYPE_INCR, 'update during backup', {strOptionalParam => '--stop-auto --buffer-size=32768 --delta'}); # Ensure the check command runs properly with a tablespace unless there is a bogus host if (!$oHostBackup->bogusHost()) @@ -702,9 +690,7 @@ sub run if ($bTestLocal) { # Expect failure because postmaster.pid exists - $oHostDbMaster->restore( - 'postmaster running', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), - {iExpectedExitStatus => ERROR_POSTMASTER_RUNNING}); + $oHostDbMaster->restore('postmaster running', 'latest', {iExpectedExitStatus => ERROR_POSTMASTER_RUNNING}); } $oHostDbMaster->clusterStop(); @@ -712,8 +698,7 @@ sub run if ($bTestLocal) { # Expect failure because db path is not empty - $oHostDbMaster->restore( - 'path not empty', cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY}); + $oHostDbMaster->restore('path not empty', 'latest', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY}); } # Drop and recreate db path @@ -726,7 +711,7 @@ sub run # Now the restore should work $oHostDbMaster->restore( - undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + undef, 'latest', {strOptionalParam => ($bTestLocal ? ' --db-include=test2 --db-include=test3' : '') . ' --buffer-size=16384'}); # Test that the first database has not been restored since --db-include did not include test1 @@ -784,8 +769,8 @@ sub run { # Backup info will have the catalog number my $oBackupInfo = new pgBackRest::Common::Ini( - storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO), - {bLoad => false, strContent => ${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)}}); + storageRepo(), $oHostBackup->repoBackupPath(FILE_BACKUP_INFO), + {bLoad => false, strContent => ${storageRepo()->get($oHostBackup->repoBackupPath(FILE_BACKUP_INFO))}}); # Construct the special path $strTablespacePath .= @@ -899,8 +884,7 @@ sub run storageTest()->put($oHostDbMaster->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL); } - $oHostDbMaster->restore( - undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE}); + $oHostDbMaster->restore(undef, 'latest', {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE}); $oHostDbMaster->clusterStart(); $oHostDbMaster->sqlSelectOneTest('select message from test', $strXidMessage); @@ -916,7 +900,7 @@ sub run $oHostDbMaster->clusterStop(); $oHostDbMaster->restore( - undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + undef, 'latest', {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_TIME, strTarget => $strTimeTarget, strTargetAction => $oHostDbMaster->pgVersion() >= PG_VERSION_91 ? 'promote' : undef, strTargetTimeline => $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'current' : undef, @@ -952,7 +936,7 @@ sub run $oHostDbMaster->clusterStop(); $oHostDbMaster->restore( - undef, cfgDefOptionDefault(CFGCMD_RESTORE, CFGOPT_SET), + undef, 'latest', {bDelta => true, bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_NAME, strTarget => $strNameTarget, strTargetAction => 'promote', strTargetTimeline => $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'current' : undef}); @@ -997,15 +981,13 @@ sub run # Incr backup - make sure a --no-online backup fails #----------------------------------------------------------------------------------------------------------------------- $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on --no-' . cfgOptionName(CFGOPT_ONLINE), - {iExpectedExitStatus => ERROR_POSTMASTER_RUNNING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)}); + CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on --no-online', + {iExpectedExitStatus => ERROR_POSTMASTER_RUNNING, strOptionalParam => '--no-online'}); # Incr backup - allow --no-online backup to succeed with --force #----------------------------------------------------------------------------------------------------------------------- $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, - 'succeed on --no-' . cfgOptionName(CFGOPT_ONLINE) . ' with --' . cfgOptionName(CFGOPT_FORCE), - {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)}); + CFGOPTVAL_BACKUP_TYPE_INCR, 'succeed on --no-online with --force', {strOptionalParam => '--no-online --force'}); } # Stanza-delete --force without access to pgbackrest on database host @@ -1017,8 +999,8 @@ sub run { $oHostDbMaster->stop(); $oHostBackup->stop({strStanza => $self->stanza}); - $oHostBackup->stanzaDelete("delete stanza with --force when pgbackrest on pg host not accessible", - {strOptionalParam => ' --' . cfgOptionName(CFGOPT_FORCE)}); + $oHostBackup->stanzaDelete( + "delete stanza with --force when pgbackrest on pg host not accessible", {strOptionalParam => ' --force'}); $oHostDbMaster->start(); $oHostBackup->start(); } diff --git a/test/test.pl b/test/test.pl index 7189d1198..1e274fe49 100755 --- a/test/test.pl +++ b/test/test.pl @@ -81,10 +81,9 @@ test.pl [options] --c-only only run C tests --container-only only run tests that must be run in a container --gen-only only run auto-generation - --gen-libc generate libc code to embed in the binary --no-gen do not run code generation --code-count generate code counts - --smart perform libc/package builds only when source timestamps have changed + --smart perform bin/package builds only when source timestamps have changed --no-package do not build packages --dev --smart --no-package --no-optimize --dev-test --no-package @@ -155,7 +154,6 @@ my $bNoCoverage = false; my $bCOnly = false; my $bContainerOnly = false; my $bGenOnly = false; -my $bGenLibC = false; my $bNoGen = false; my $bCodeCount = false; my $bSmart = false; @@ -206,7 +204,6 @@ GetOptions ('q|quiet' => \$bQuiet, 'c-only' => \$bCOnly, 'container-only' => \$bContainerOnly, 'gen-only' => \$bGenOnly, - 'gen-libc' => \$bGenLibC, 'no-gen' => \$bNoGen, 'code-count' => \$bCodeCount, 'smart' => \$bSmart, @@ -369,6 +366,7 @@ eval # Get the base backrest path my $strBackRestBase = dirname(dirname(abs_path($0))); + my $strVagrantPath = "${strBackRestBase}/test/.vagrant"; my $oStorageBackRest = new pgBackRestTest::Common::Storage( $strBackRestBase, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); @@ -550,28 +548,9 @@ eval } } - # Auto-generate Perl code - #----------------------------------------------------------------------------------------------------------------------- - use lib dirname(dirname($0)) . '/libc/build/lib'; - use pgBackRestLibC::Build; - - if (!$bSmart || grep(/^(build|libc\/build)\//, @stryModifiedList)) - { - errorDefineLoad(${$oStorageBackRest->get("build/error.yaml")}); - - my @stryBuilt = buildXsAll("${strBackRestBase}/libc"); - &log(INFO, " autogenerated Perl code: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); - - if (@stryBuilt) - { - push(@stryBuiltAll, @stryBuilt); - push(@stryModifiedList, @stryBuilt); - } - } - # Auto-generate C Makefile #----------------------------------------------------------------------------------------------------------------------- - if (!$bSmart || grep(/^(src|libc)\//, @stryModifiedList)) + if (!$bSmart || grep(/^src\//, @stryModifiedList)) { my @stryBuilt; my $strBuilt = 'src/Makefile.in'; @@ -713,8 +692,7 @@ eval #--------------------------------------------------------------------------------------------------------------------------- my $oyTestRun; my $bBinRequired = $bBuildOnly; - my $bLibCHostRequired = $bBuildOnly; - my $bLibCVmRequired = $bBuildOnly; + my $bHostBinRequired = $bBuildOnly; # Only get the test list when they can run if (!$bBuildOnly) @@ -732,37 +710,26 @@ eval $bBinRequired = true; } - # Host LibC required if a Perl test + # Host bin required if a Perl test if (!$hTest->{&TEST_C}) { - $bLibCHostRequired = true; - } - - # VM LibC required if Perl and not an integration test - if (!$hTest->{&TEST_C} && !$hTest->{&TEST_INTEGRATION}) - { - $bLibCVmRequired = true; + $bHostBinRequired = true; } } } my $strBuildRequired; - if ($bBinRequired || $bLibCHostRequired || $bLibCVmRequired) + if ($bBinRequired || $bHostBinRequired) { if ($bBinRequired) { $strBuildRequired = "bin"; } - if ($bLibCHostRequired) + if ($bHostBinRequired) { - $strBuildRequired .= ", libc host"; - } - - if ($bLibCVmRequired) - { - $strBuildRequired .= ", libc vm"; + $strBuildRequired .= ", bin host"; } } else @@ -777,9 +744,8 @@ eval if (!$bDryRun) { my $oVm = vmGet(); - my $strVagrantPath = "${strBackRestBase}/test/.vagrant"; my $lTimestampLast; - my @stryBinSrcPath = ('src', 'libc'); + my @stryBinSrcPath = ('src'); my $strBinPath = "${strVagrantPath}/bin"; my $rhBinBuild = {}; @@ -794,6 +760,12 @@ eval my $bLogDetail = $strLogLevel eq 'detail'; my @stryBuildVm = $strVm eq VM_ALL ? VM_LIST : ($strVm); + # Build binary for the host + if ($bHostBinRequired) + { + push(@stryBuildVm, VM_NONE); + } + foreach my $strBuildVM (@stryBuildVm) { my $strBuildPath = "${strBinPath}/${strBuildVM}/src"; @@ -802,7 +774,7 @@ eval # Build configure/compile options and see if they have changed from the previous build my $strCFlags = - "-Wfatal-errors -g -fPIC -D_FILE_OFFSET_BITS=64" . + "-Wfatal-errors -g" . (vmWithBackTrace($strBuildVM) && $bBackTrace ? ' -DWITH_BACKTRACE' : '') . ($bDebugTestTrace ? ' -DDEBUG_TEST_TRACE' : ''); my $strLdFlags = vmWithBackTrace($strBuildVM) && $bBackTrace ? '-lbacktrace' : ''; @@ -872,114 +844,6 @@ eval } } - # Build the C Library - #----------------------------------------------------------------------------------------------------------------------- - if ($bLibCHostRequired || $bLibCVmRequired) - { - my $strLibCPath = "${strVagrantPath}/bin"; - - # Loop through VMs to do the C Library builds - my $bLogDetail = $strLogLevel eq 'detail'; - my @stryBuildVm = (); - - if ($strVm eq VM_ALL) - { - @stryBuildVm = $bLibCVmRequired ? VM_LIST : ($strVmHost); - } - else - { - @stryBuildVm = $bLibCVmRequired && $strVmHost ne $strVm ? ($strVmHost, $strVm) : ($strVmHost); - } - - foreach my $strBuildVM (@stryBuildVm) - { - my $strBuildPath = "${strLibCPath}/${strBuildVM}/libc"; - my $bContainerExists = $strBuildVM ne $strVmHost && $strBuildVM ne VM_NONE; - my $strConfigOptions = (vmDebugIntegration($strBuildVM) ? ' --enable-test' : ''); - - my $strLibCSmart = "${strBuildPath}/blib/arch/auto/pgBackRest/LibC/LibC.so"; - my $bRebuild = !$bSmart; - - # Rebuild if the modification time of the smart file does equal the last changes in source paths - if ($bSmart) - { - if (!$oStorageBackRest->exists($strLibCSmart) || - $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast) - { - &log(INFO, " libc dependencies have changed for ${strBuildVM}, rebuilding..."); - - $bRebuild = true; - } - } - - if ($bRebuild) - { - &log(INFO, " build test library for ${strBuildVM} (${strBuildPath})"); - - if (!$rhBinBuild->{$strBuildVM}) - { - foreach my $strBinSrcPath (@stryBinSrcPath) - { - $oStorageBackRest->pathCreate( - "${strBinPath}/${strBuildVM}/${strBinSrcPath}", {bIgnoreExists => true, bCreateParent => true}); - } - - executeTest( - "rsync -rt" . (!$bSmart ? " --delete-excluded" : '') . - " --include=" . join('/*** --include=', @stryBinSrcPath) . '/*** --exclude=*' . - " ${strBackRestBase}/ ${strBinPath}/${strBuildVM}"); - } - - # Can't reuse any object files in the libc dir because it does not have proper dependencies - executeTest( - "rsync -rt --exclude=Makefile --delete ${strBackRestBase}/libc/ ${strLibCPath}/${strBuildVM}/libc"); - - # It's very expensive to rebuild the Makefile so make sure it has actually changed - my $bMakeRebuild = - !$oStorageBackRest->exists("${strBuildPath}/Makefile") || - ($oStorageBackRest->info("${strBackRestBase}/libc/Makefile.PL")->mtime > - $oStorageBackRest->info("${strBuildPath}/Makefile.PL")->mtime); - - if ($bContainerExists) - { - executeTest( - "docker run -itd -h test-build --name=test-build" . - " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-build", - {bSuppressStdErr => true}); - } - - if ($bMakeRebuild) - { - &log(INFO, " rebuild test library Makefile for ${strBuildVM}"); - - executeTest( - ($bContainerExists ? "docker exec -i test-build bash -c '" : '') . - "cd ${strBuildPath} && perl ${strBuildPath}/Makefile.PL INSTALLMAN1DIR=none INSTALLMAN3DIR=none" . - ($bContainerExists ? "'" : ''), - {bSuppressStdErr => true, bShowOutputAsync => $bLogDetail}); - } - - if (!$oStorageBackRest->exists("${strLibCPath}/${strBuildVM}/src/Makefile")) - { - executeTest( - ($bContainerExists ? 'docker exec -i test-build ' : '') . - "bash -c 'cd ${strLibCPath}/${strBuildVM}/src && ./configure${strConfigOptions}'", - {bShowOutputAsync => $bLogDetail}); - } - - executeTest( - ($bContainerExists ? 'docker exec -i test-build ' : '') . - "make -j ${iBuildMax} --silent --directory ${strBuildPath}", - {bShowOutputAsync => $bLogDetail}); - - if ($bContainerExists) - { - executeTest("docker rm -f test-build"); - } - } - } - } - # Build the package #----------------------------------------------------------------------------------------------------------------------- if (!$bNoPackage && $strVm ne VM_NONE) @@ -1125,7 +989,7 @@ eval executeTest( "tar --transform='s_^_pgbackrest-release-${strVersionBase}/_'" . " -czf ${strBuildPath}/SOURCES/${strVersionBase}.tar.gz -C ${strBackRestBase}" . - " lib libc src LICENSE"); + " src LICENSE"); # Copy package files executeTest( @@ -1463,13 +1327,6 @@ eval ################################################################################################################################ # Runs tests ################################################################################################################################ - push( - @INC, - "${strBackRestBase}/test/.vagrant/bin/" . - ((testDefModuleTest($stryModule[0], $stryModuleTest[0]))->{&TESTDEF_CONTAINER} ? $strVm : $strVmHost) . - "/libc/blib/arch"); - buildLoadLibC(); - my $iRun = 0; # Create host group for containers @@ -1487,7 +1344,7 @@ eval $strBackRestBase, # Base backrest directory $strTestPath, # Path where the tests will run '/usr/bin/' . PROJECT_EXE, # Path to the backrest executable - "${strBackRestBase}/bin/" . PROJECT_EXE, # Path to the backrest Perl helper + "${strVagrantPath}/bin/" . VM_NONE . '/src/' . PROJECT_EXE, # Path to the backrest Perl storage helper $strPgVersion ne 'minimal' ? $strPgSqlBin: undef, # Pg bin path $strPgVersion ne 'minimal' ? $strPgVersion: undef, # Pg version $stryModule[0], $stryModuleTest[0], \@iyModuleTestRun, # Module info