db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-06-20 01:17:49 +02:00
Code Issues Releases Activity

More paths through copy() working.

Browse Source
This commit is contained in:
David Steele
2014-06-05 22:42:47 -04:00
parent ca6b435c4f
commit e44cd6c1ca
3 changed files with 84 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/pg_backrest_command.pl
+16 -10
View File
@@ -46,14 +46,17 @@ my $strExpression = undef; # Expression to use for filtering (undef =
my $strPermission = undef; # Permission when creating directory or file (undef = default)
my $strSort = undef; # Sort order (undef = forward)
GetOptions ("ignore-missing" => \$bIgnoreMissing,
if (!GetOptions ("ignore-missing" => \$bIgnoreMissing,
"dest-path-create" => \$bDestinationPathCreate,
"compress" => \$bCompress,
"uncompress" => \$bUncompress,
"expression=s" => \$strExpression,
"permission=s" => \$strPermission,
"sort=s" => \$strSort)
or die("Error in command line arguments\n");
"sort=s" => \$strSort))
{
print(STDERR "error in command line arguments");
exit COMMAND_ERR_PARAM;
}
####################################################################################################################################
# START MAIN
@@ -67,13 +70,15 @@ my $strOperation = $ARGV[0];
# Validate the operation
if (!defined($strOperation))
{
confess &log(ERROR, "operation is not defined");
print(STDERR "operation is not defined");
exit COMMAND_ERR_PARAM;
}
# Make sure compress and uncompress are not both set
if ($bCompress && $bUncompress)
{
confess "compress and uncompress options cannot both be set";
print(STDERR "compress and uncompress options cannot both be set");
exit COMMAND_ERR_PARAM;
}
# Create the file object
@@ -248,7 +253,7 @@ if ($strOperation eq OP_COPY_IN)
# Open the source file
my $hIn;
if (!open($hIn, "<", ${strFileSource}))
{
$strError = $!;
@@ -269,7 +274,7 @@ if ($strOperation eq OP_COPY_IN)
{
$oFile->pipe($hIn, *STDOUT, $bCompress && !$bAlreadyCompressed, $bUncompress && $bAlreadyCompressed);
};
$strError = $@;
# Close the input file
@@ -298,7 +303,7 @@ if ($strOperation eq OP_COPY_OUT)
confess "destination file must be specified for ${strOperation} operation";
}
# Determine of the file needs compression extension
# Determine if the file needs compression extension
if ($bCompress && $strFileDestination !~ "^.*\.$oFile->{strCompressExtension}\$")
{
$strFileDestination .= "." . $oFile->{strCompressExtension};
@@ -309,7 +314,7 @@ if ($strOperation eq OP_COPY_OUT)
# Open the destination file
my $hOut;
if (!open($hOut, ">", ${strFileDestination}))
{
$strError = $!;
@@ -354,4 +359,5 @@ if ($strOperation eq OP_PATH_CREATE)
exit 0;
}
confess &log(ERROR, "invalid operation ${strOperation}");
print(STDERR "invalid operation ${strOperation}");
exit COMMAND_ERR_PARAM;
pg_backrest_file.pm
+46 -51
View File
@@ -18,15 +18,16 @@ use File::stat;
use Fcntl ':mode';
use IO::Compress::Gzip qw(gzip $GzipError);
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
use IO::String;
use lib dirname($0);
use pg_backrest_utility;
use Exporter qw(import);
our @EXPORT = qw(PATH_ABSOLUTE PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_ABSOLUTE
PATH_BACKUP_CLUSTERPATH_BACKUP_TMP PATH_BACKUP_ARCHIVE
COMMAND_ERR_FILE_MISSING COMMAND_ERR_FILE_READ COMMAND_ERR_FILE_MOVE
COMMAND_ERR_FILE_TYPE COMMAND_ERR_LINK_READ COMMAND_ERR_PATH_MISSING COMMAND_ERR_PATH_CREATE);
PATH_BACKUP_CLUSTERPATH_BACKUP_TMP PATH_BACKUP_ARCHIVE
COMMAND_ERR_FILE_MISSING COMMAND_ERR_FILE_READ COMMAND_ERR_FILE_MOVE COMMAND_ERR_FILE_TYPE
COMMAND_ERR_LINK_READ COMMAND_ERR_PATH_MISSING COMMAND_ERR_PATH_CREATE COMMAND_ERR_PARAM);
# Extension and permissions
has strCompressExtension => (is => 'ro', default => 'gz');
@@ -35,9 +36,6 @@ has strDefaultFilePermission => (is => 'ro', default => '0640');
# Command strings
has strCommand => (is => 'bare');
has strCommandCompress => (is => 'bare');
has strCommandDecompress => (is => 'bare');
has strCommandCat => (is => 'bare', default => 'cat %file%');
# Lock path
has strLockPath => (is => 'bare');
@@ -54,7 +52,7 @@ has strBackupPath => (is => 'bare'); # Backup base path
has strBackupClusterPath => (is => 'bare'); # Backup cluster path
# Process flags
has bNoCompression => (is => 'bare');
has bCompress => (is => 'bare');
has strStanza => (is => 'bare');
has iThreadIdx => (is => 'bare');
@@ -69,7 +67,8 @@ use constant
COMMAND_ERR_FILE_TYPE => 4,
COMMAND_ERR_LINK_READ => 5,
COMMAND_ERR_PATH_MISSING => 6,
COMMAND_ERR_PATH_CREATE => 7
COMMAND_ERR_PATH_CREATE => 7,
COMMAND_ERR_PARAM => 8
};
####################################################################################################################################
@@ -121,10 +120,10 @@ sub BUILD
my $strOptionSSHRequestTTY = "RequestTTY=yes";
my $strOptionSSHCompression = "Compression=no";
if ($self->{bNoCompression})
{
$strOptionSSHCompression = "Compression=yes";
}
# if ($self->{bNoCompression})
# {
# $strOptionSSHCompression = "Compression=yes";
# }
# Connect SSH object if backup host is defined
if (!defined($self->{oBackupSSH}) && defined($self->{strBackupHost}))
@@ -132,7 +131,6 @@ sub BUILD
&log(TRACE, "connecting to backup ssh host " . $self->{strBackupHost});
$self->{oBackupSSH} = Net::OpenSSH->new($self->{strBackupHost}, timeout => 300, user => $self->{strBackupUser},
# default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
master_opts => [-o => $strOptionSSHCompression, -o => $strOptionSSHRequestTTY]);
$self->{oBackupSSH}->error and confess &log(ERROR, "unable to connect to $self->{strBackupHost}: " . $self->{oBackupSSH}->error);
}
@@ -143,7 +141,6 @@ sub BUILD
&log(TRACE, "connecting to database ssh host $self->{strDbHost}");
$self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, timeout => 300, user => $self->{strDbUser},
# default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
master_opts => [-o => $strOptionSSHCompression, -o => $strOptionSSHRequestTTY]);
$self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error);
}
@@ -164,16 +161,13 @@ sub clone
strDefaultPathPermission => $self->{strDefaultPathPermission},
strDefaultFilePermission => $self->{strDefaultFilePermission},
strCommand => $self->{strCommand},
strCommandCompress => $self->{strCommandCompress},
strCommandDecompress => $self->{strCommandDecompress},
strCommandCat => $self->{strCommandCat},
strDbUser => $self->{strDbUser},
strDbHost => $self->{strDbHost},
strBackupUser => $self->{strBackupUser},
strBackupHost => $self->{strBackupHost},
strBackupPath => $self->{strBackupPath},
strBackupClusterPath => $self->{strBackupClusterPath},
bNoCompression => $self->{bNoCompression},
bCompress => $self->{bCompress},
strStanza => $self->{strStanza},
iThreadIdx => $iThreadIdx,
strLockPath => $self->{strLockPath}
@@ -186,14 +180,14 @@ sub clone
# sub error_get
# {
# my $self = shift;
#
#
# my $strErrorFile = $self->path_get(PATH_LOCK_ERR, "file");
#
#
# open my $hFile, '<', $strErrorFile or return "error opening ${strErrorFile} to read STDERR output";
#
#
# my $strError = do {local $/; <$hFile>};
# close $hFile;
#
#
# return trim($strError);
# }
@@ -203,7 +197,7 @@ sub clone
# sub error_clear
# {
# my $self = shift;
#
#
# unlink($self->path_get(PATH_LOCK_ERR, "file"));
# }
@@ -408,7 +402,6 @@ sub remote_get
confess &log(ASSERT, "path type ${strPathType} does not have a defined ssh object");
}
####################################################################################################################################
# LINK_CREATE !!! NEEDS TO BE CONVERTED
####################################################################################################################################
@@ -647,13 +640,12 @@ sub move
}
}
####################################################################################################################################
# PIPE Function
#
# Copies data from one file handle to another, optionally compressing or decompressing the data in stream.
####################################################################################################################################
sub pipe()
sub pipe
{
my $self = shift;
my $hIn = shift;
@@ -683,7 +675,7 @@ sub pipe()
my $strBuffer;
my $iResultRead;
my $iResultWrite;
# Read from the input handle
while (($iResultRead = sysread($hIn, $strBuffer, BLOCK_SIZE)) != 0)
{
@@ -696,7 +688,7 @@ sub pipe()
{
# Write to the output handle
$iResultWrite = syswrite($hOut, $strBuffer, $iResultRead);
if (!defined($iResultWrite) || $iResultWrite != $iResultRead)
{
confess $!;
@@ -711,7 +703,7 @@ sub pipe()
# COPY
#
# Copies a file from one location to another:
#
#
# * source and destination can be local or remote
# * wire and output compression/decompression are supported
# * intermediate temp files are used to prevent partial copies
@@ -732,7 +724,7 @@ sub copy
my $strPermission = shift;
# Set defaults
$bCompress = defined($bCompress) ? $bCompress : defined($self->{bNoCompression}) ? !$self->{bNoCompression} : true;
$bCompress = defined($bCompress) ? $bCompress : defined($self->{bCompress}) ? $self->{bCompress} : true;
$bIgnoreMissingSource = defined($bIgnoreMissingSource) ? $bIgnoreMissingSource : false;
$bPathCreate = defined($bPathCreate) ? $bPathCreate : false;
@@ -745,9 +737,16 @@ sub copy
my $strDestinationTmpOp = $self->path_get($strDestinationPathType, $strDestinationFile, true);
my $strError;
# Determine if the file needs compression extension
if ($bCompress && $strDestinationOp !~ "^.*\.$self->{strCompressExtension}\$")
{
$strDestinationOp .= "." . $self->{strCompressExtension};
}
# Output trace info
&log(TRACE, "${strErrorPrefix}:" . ($bSourceRemote ? " remote" : " local") . " ${strSourcePathType}:${strSourceFile}" .
" to " . ($bDestinationRemote ? " remote" : " local") . " ${strDestinationPathType}:${strDestinationFile}");
" to" . ($bDestinationRemote ? " remote" : " local") . " ${strDestinationPathType}:${strDestinationFile}" .
", compress = " . ($bCompress ? "true" : "false"));
# If source or destination are remote
if ($bSourceRemote || $bDestinationRemote)
@@ -783,12 +782,10 @@ sub copy
# Else if source is local and destination is remote
elsif (!$bSourceRemote && $bDestinationRemote)
{
# !!! SOURCE LOCAL DESTINATION REMOTE COPY NOT YET IMPLEMENTED
return false;
# Build the command string
$strCommand = $self->{strCommand} .
" --compress copy_out ${strDestinationOp}";
($bCompress ? " --compress" : " --uncompress") .
" copy_out ${strDestinationOp}";
# Open source file for reading
open($hFile, "<", $strSourceOp)
@@ -806,6 +803,8 @@ sub copy
return false;
}
&log(TRACE, "${strErrorPrefix} command:" . $strCommand);
# Execute the ssh command
my ($hIn, $hOut, $hErr, $pId) = $oSSH->open3($strCommand)
or confess &log("unable to execute ssh '${strCommand}': " . $self->error_get());
@@ -821,27 +820,23 @@ sub copy
$self->pipe($hFile, $hIn, $bCompress, !$bCompress);
}
# Read STDERR into a string
my $strError = "";
open my ($hErrOut), '>', $strError;
$self->pipe($hErr, $hErrOut);
close($hErrOut);
# Close hIn
close($hIn);
# Read STDOUT into a string
my $strOutput = "";
open my ($hOutString), '>', $strOutput;
$self->pipe($hOut, $hOutString);
close($hOutString);
# Read STDERR into a string
my $strError;
my $hErrString = IO::String->new($strError);
$self->pipe($hErr, $hErrString);
# Wait for the process to finish and report any errors
waitpid($pId, 0);
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
# close($hErrString);
if ($iExitStatus != 0)
{
confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus . ": " . $strError);
confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus . ": " .
(defined($strError) ? $strError : "[unknown]"));
}
# Close the destination file handle
@@ -864,15 +859,15 @@ sub copy
system("chmod ${strPermission} ${strDestinationTmpOp}") == 0
or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmpOp}");
}
# Set the file modification time if required (this only works locally for now)
if (defined($lModificationTime))
{
utime($lModificationTime, $lModificationTime, $strDestinationTmpOp)
or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}");
}
# Move the file from tmp to final destination
$self->move($self->path_type_get($strDestinationPathType) . ":absolute", $strDestinationTmpOp,
$self->path_type_get($strDestinationPathType) . ":absolute", $strDestinationOp, $bPathCreate);
test/lib/BackRestTest/FileTest.pm
+22 -50
View File
@@ -26,7 +26,7 @@ our @EXPORT = qw(BackRestFileTest);
sub BackRestFileTest
{
my $strTest = shift;
# If no test was specified, then run them all
if (!defined($strTest))
{
@@ -737,29 +737,6 @@ sub BackRestFileTest
if ($strTest eq 'all' || $strTest eq 'copy')
{
$iRun = 0;
# my $strLockPath = dirname(abs_path($0)) . "/test/lock";
#
#
# my $oFile = pg_backrest_file->new
# (
# strStanza => "db",
# bNoCompression => false,
# strBackupClusterPath => undef,
# strBackupPath => ${strTestPath},
# strBackupHost => $strHost,
# strBackupUser => $strUser,
# strDbHost => undef,
# strDbUser => undef,
# strLockPath => $strLockPath
# );
#
# my $strSourceFile = "${strTestPath}/backup/test.txt";
# my $strDestinationFile = "${strTestPath}/db/test.txt";
#
# system("echo 'TESTDATA' > ${strSourceFile}");
#
# $oFile->copy(PATH_BACKUP_ABSOLUTE, $strSourceFile, PATH_DB_ABSOLUTE, $strDestinationFile, undef, false);
for (my $bBackupRemote = 0; $bBackupRemote <= 1; $bBackupRemote++)
{
@@ -771,7 +748,7 @@ sub BackRestFileTest
{
next;
}
# Loop through destination compression
for (my $bDestinationCompressed = 0; $bDestinationCompressed <= 1; $bDestinationCompressed++)
{
@@ -779,7 +756,7 @@ sub BackRestFileTest
(
strStanza => "db",
strCommand => $strCommand,
bNoCompression => !$bDestinationCompressed,
bCompress => $bDestinationCompressed,
strBackupClusterPath => undef,
strBackupPath => ${strTestPath},
strBackupHost => $bBackupRemote ? $strHost : undef,
@@ -787,14 +764,14 @@ sub BackRestFileTest
strDbHost => $bDbRemote ? $strHost : undef,
strDbUser => $bDbRemote ? $strUser : undef
);
for (my $bSourceCompressed = 0; $bSourceCompressed <= 1; $bSourceCompressed++)
{
for (my $bSourcePathType = 0; $bSourcePathType <= 1; $bSourcePathType++)
{
my $strSourcePathType = $bSourcePathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
my $strSourcePath = $bSourcePathType ? "db" : "backup";
for (my $bDestinationPathType = 0; $bDestinationPathType <= 1; $bDestinationPathType++)
{
my $strDestinationPathType = $bDestinationPathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
@@ -825,7 +802,6 @@ sub BackRestFileTest
{
system("echo 'TESTDATA' > ${strSourceFile}");
}
# Create the file object based on current values
# Run file copy in an eval block because some errors are expected
my $bReturn;
@@ -840,29 +816,29 @@ sub BackRestFileTest
if ($@)
{
# Different remote and destination with different path types should error
if (($bBackupRemote || $bDbRemote) && ($strSourcePathType ne $strDestinationPathType))
{
print " different source and remote for same path not supported\n";
next;
}
# if (($bBackupRemote || $bDbRemote) && ($strSourcePathType ne $strDestinationPathType))
# {
# print " different source and remote for same path not supported\n";
# next;
# }
# If the error was intentional, then also continue
# elsif ($bError)
# {
# my $strError = $oFile->error_get();
#
#
# if (!defined($strError) || ($strError eq ''))
# {
# confess 'no error message returned';
# }
#
#
# print " error raised: ${strError}\n";
# next;
# }
# Else this is an unexpected error
else
{
# else
# {
confess $@;
}
# }
}
# elsif ($bError)
# {
@@ -879,12 +855,12 @@ sub BackRestFileTest
# else
# {
# my $strError = $oFile->error_get();
#
#
# if (!defined($strError) || ($strError eq ''))
# {
# confess 'no error message returned';
# }
#
#
# print " error returned: ${strError}\n";
# next;
# }
@@ -896,15 +872,15 @@ sub BackRestFileTest
# {
# confess "error was returned when no error generated";
# }
#
#
# print " true was returned\n";
# }
# Check for errors after copy
# if ($bDestinationCompressed)
# {
# $strDestinationFile .= ".gz";
# }
if ($bDestinationCompressed)
{
$strDestinationFile .= ".gz";
}
if ($bReturn)
{
@@ -913,10 +889,6 @@ sub BackRestFileTest
confess "could not find destination file ${strDestinationFile}";
}
}
else
{
&log(INFO, "Not yet implemented");
}
}
}
}