mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-01-30 05:39:12 +02:00
June Hackathon: copy_in and copy_out functions written, some paths through File->copy() working, basic unit tests setup.
This commit is contained in:
David Steele
parent
c2dd53f908
commit
ca6b435c4f
@ -38,6 +38,7 @@ Simple Postgres Backup and Restore
|
|||||||
* IPC::Open3
|
* IPC::Open3
|
||||||
* Digest::SHA
|
* Digest::SHA
|
||||||
* IO::Compress::Gzip
|
* IO::Compress::Gzip
|
||||||
|
* IO::Uncompress::Gunzip
|
||||||
|
|
||||||
## release notes
|
## release notes
|
||||||
|
|
||||||
|
|||||||
@ -30,6 +30,8 @@ use constant
|
|||||||
OP_MANIFEST => "manifest",
|
OP_MANIFEST => "manifest",
|
||||||
OP_COMPRESS => "compress",
|
OP_COMPRESS => "compress",
|
||||||
OP_MOVE => "move",
|
OP_MOVE => "move",
|
||||||
|
OP_COPY_OUT => "copy_out",
|
||||||
|
OP_COPY_IN => "copy_in",
|
||||||
OP_PATH_CREATE => "path_create"
|
OP_PATH_CREATE => "path_create"
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -38,12 +40,16 @@ use constant
|
|||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
my $bIgnoreMissing = false; # Ignore errors due to missing file
|
my $bIgnoreMissing = false; # Ignore errors due to missing file
|
||||||
my $bDestinationPathCreate = false; # Create destination path if it does not exist
|
my $bDestinationPathCreate = false; # Create destination path if it does not exist
|
||||||
|
my $bCompress = false; # Compress output
|
||||||
|
my $bUncompress = false; # Uncompress output
|
||||||
my $strExpression = undef; # Expression to use for filtering (undef = no filtering)
|
my $strExpression = undef; # Expression to use for filtering (undef = no filtering)
|
||||||
my $strPermission = undef; # Permission when creating directory or file (undef = default)
|
my $strPermission = undef; # Permission when creating directory or file (undef = default)
|
||||||
my $strSort = undef; # Sort order (undef = forward)
|
my $strSort = undef; # Sort order (undef = forward)
|
||||||
|
|
||||||
GetOptions ("ignore-missing" => \$bIgnoreMissing,
|
GetOptions ("ignore-missing" => \$bIgnoreMissing,
|
||||||
"dest-path-create" => \$bDestinationPathCreate,
|
"dest-path-create" => \$bDestinationPathCreate,
|
||||||
|
"compress" => \$bCompress,
|
||||||
|
"uncompress" => \$bUncompress,
|
||||||
"expression=s" => \$strExpression,
|
"expression=s" => \$strExpression,
|
||||||
"permission=s" => \$strPermission,
|
"permission=s" => \$strPermission,
|
||||||
"sort=s" => \$strSort)
|
"sort=s" => \$strSort)
|
||||||
@ -64,6 +70,12 @@ if (!defined($strOperation))
|
|||||||
confess &log(ERROR, "operation is not defined");
|
confess &log(ERROR, "operation is not defined");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Make sure compress and uncompress are not both set
|
||||||
|
if ($bCompress && $bUncompress)
|
||||||
|
{
|
||||||
|
confess "compress and uncompress options cannot both be set";
|
||||||
|
}
|
||||||
|
|
||||||
# Create the file object
|
# Create the file object
|
||||||
my $oFile = pg_backrest_file->new();
|
my $oFile = pg_backrest_file->new();
|
||||||
|
|
||||||
@ -218,6 +230,113 @@ if ($strOperation eq OP_MOVE)
|
|||||||
exit 0;
|
exit 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
####################################################################################################################################
|
||||||
|
# COPY_IN Command
|
||||||
|
####################################################################################################################################
|
||||||
|
if ($strOperation eq OP_COPY_IN)
|
||||||
|
{
|
||||||
|
my $strFileSource = $ARGV[1];
|
||||||
|
|
||||||
|
# Make sure the source file is defined
|
||||||
|
if (!defined($strFileSource))
|
||||||
|
{
|
||||||
|
confess "source file must be specified for ${strOperation} operation";
|
||||||
|
}
|
||||||
|
|
||||||
|
# Variable to hold errors
|
||||||
|
my $strError;
|
||||||
|
|
||||||
|
# Open the source file
|
||||||
|
my $hIn;
|
||||||
|
|
||||||
|
if (!open($hIn, "<", ${strFileSource}))
|
||||||
|
{
|
||||||
|
$strError = $!;
|
||||||
|
|
||||||
|
unless (-e $strFileSource)
|
||||||
|
{
|
||||||
|
print(STDERR "${strFileSource} does not exist");
|
||||||
|
exit COMMAND_ERR_FILE_MISSING;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
# Determine if the file is already compressed
|
||||||
|
my $bAlreadyCompressed = ($strFileSource =~ "^.*\.$oFile->{strCompressExtension}\$");
|
||||||
|
|
||||||
|
# Copy the file to STDOUT
|
||||||
|
eval
|
||||||
|
{
|
||||||
|
$oFile->pipe($hIn, *STDOUT, $bCompress && !$bAlreadyCompressed, $bUncompress && $bAlreadyCompressed);
|
||||||
|
};
|
||||||
|
|
||||||
|
$strError = $@;
|
||||||
|
|
||||||
|
# Close the input file
|
||||||
|
close($hIn);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($strError)
|
||||||
|
{
|
||||||
|
print(STDERR "${strFileSource} could not be read: ${strError}");
|
||||||
|
exit COMMAND_ERR_FILE_READ;
|
||||||
|
}
|
||||||
|
|
||||||
|
exit 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
####################################################################################################################################
|
||||||
|
# COPY_OUT Command
|
||||||
|
####################################################################################################################################
|
||||||
|
if ($strOperation eq OP_COPY_OUT)
|
||||||
|
{
|
||||||
|
my $strFileDestination = $ARGV[1];
|
||||||
|
|
||||||
|
# Make sure the source file is defined
|
||||||
|
if (!defined($strFileDestination))
|
||||||
|
{
|
||||||
|
confess "destination file must be specified for ${strOperation} operation";
|
||||||
|
}
|
||||||
|
|
||||||
|
# Determine of the file needs compression extension
|
||||||
|
if ($bCompress && $strFileDestination !~ "^.*\.$oFile->{strCompressExtension}\$")
|
||||||
|
{
|
||||||
|
$strFileDestination .= "." . $oFile->{strCompressExtension};
|
||||||
|
}
|
||||||
|
|
||||||
|
# Variable to hold errors
|
||||||
|
my $strError;
|
||||||
|
|
||||||
|
# Open the destination file
|
||||||
|
my $hOut;
|
||||||
|
|
||||||
|
if (!open($hOut, ">", ${strFileDestination}))
|
||||||
|
{
|
||||||
|
$strError = $!;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
# Copy the file from STDIN
|
||||||
|
eval
|
||||||
|
{
|
||||||
|
$strError = $oFile->pipe(*STDIN, $hOut, $bCompress, $bUncompress);
|
||||||
|
};
|
||||||
|
|
||||||
|
$strError = $@;
|
||||||
|
|
||||||
|
# Close the input file
|
||||||
|
close($hOut);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($strError)
|
||||||
|
{
|
||||||
|
print(STDERR "${strFileDestination} could not be written: ${strError}");
|
||||||
|
exit COMMAND_ERR_FILE_READ;
|
||||||
|
}
|
||||||
|
|
||||||
|
exit 0;
|
||||||
|
}
|
||||||
|
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# PATH_CREATE Command
|
# PATH_CREATE Command
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
|
|||||||
@ -17,13 +17,16 @@ use Digest::SHA;
|
|||||||
use File::stat;
|
use File::stat;
|
||||||
use Fcntl ':mode';
|
use Fcntl ':mode';
|
||||||
use IO::Compress::Gzip qw(gzip $GzipError);
|
use IO::Compress::Gzip qw(gzip $GzipError);
|
||||||
|
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
|
||||||
|
|
||||||
use lib dirname($0);
|
use lib dirname($0);
|
||||||
use pg_backrest_utility;
|
use pg_backrest_utility;
|
||||||
|
|
||||||
use Exporter qw(import);
|
use Exporter qw(import);
|
||||||
our @EXPORT = qw(PATH_ABSOLUTE PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_ABSOLUTE PATH_BACKUP_CLUSTERPATH_BACKUP_TMP
|
our @EXPORT = qw(PATH_ABSOLUTE PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_ABSOLUTE
|
||||||
PATH_BACKUP_ARCHIVE);
|
PATH_BACKUP_CLUSTERPATH_BACKUP_TMP PATH_BACKUP_ARCHIVE
|
||||||
|
COMMAND_ERR_FILE_MISSING COMMAND_ERR_FILE_READ COMMAND_ERR_FILE_MOVE
|
||||||
|
COMMAND_ERR_FILE_TYPE COMMAND_ERR_LINK_READ COMMAND_ERR_PATH_MISSING COMMAND_ERR_PATH_CREATE);
|
||||||
|
|
||||||
# Extension and permissions
|
# Extension and permissions
|
||||||
has strCompressExtension => (is => 'ro', default => 'gz');
|
has strCompressExtension => (is => 'ro', default => 'gz');
|
||||||
@ -85,6 +88,14 @@ use constant
|
|||||||
PATH_LOCK_ERR => 'lock:err'
|
PATH_LOCK_ERR => 'lock:err'
|
||||||
};
|
};
|
||||||
|
|
||||||
|
####################################################################################################################################
|
||||||
|
# File copy block size constant
|
||||||
|
####################################################################################################################################
|
||||||
|
use constant
|
||||||
|
{
|
||||||
|
BLOCK_SIZE => 8192
|
||||||
|
};
|
||||||
|
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# CONSTRUCTOR
|
# CONSTRUCTOR
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
@ -172,29 +183,29 @@ sub clone
|
|||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# ERROR_GET
|
# ERROR_GET
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
sub error_get
|
# sub error_get
|
||||||
{
|
# {
|
||||||
my $self = shift;
|
# my $self = shift;
|
||||||
|
#
|
||||||
my $strErrorFile = $self->path_get(PATH_LOCK_ERR, "file");
|
# my $strErrorFile = $self->path_get(PATH_LOCK_ERR, "file");
|
||||||
|
#
|
||||||
open my $hFile, '<', $strErrorFile or return "error opening ${strErrorFile} to read STDERR output";
|
# open my $hFile, '<', $strErrorFile or return "error opening ${strErrorFile} to read STDERR output";
|
||||||
|
#
|
||||||
my $strError = do {local $/; <$hFile>};
|
# my $strError = do {local $/; <$hFile>};
|
||||||
close $hFile;
|
# close $hFile;
|
||||||
|
#
|
||||||
return trim($strError);
|
# return trim($strError);
|
||||||
}
|
# }
|
||||||
|
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# ERROR_CLEAR
|
# ERROR_CLEAR
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
sub error_clear
|
# sub error_clear
|
||||||
{
|
# {
|
||||||
my $self = shift;
|
# my $self = shift;
|
||||||
|
#
|
||||||
unlink($self->path_get(PATH_LOCK_ERR, "file"));
|
# unlink($self->path_get(PATH_LOCK_ERR, "file"));
|
||||||
}
|
# }
|
||||||
|
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# PATH_TYPE_GET
|
# PATH_TYPE_GET
|
||||||
@ -636,194 +647,236 @@ sub move
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# COPY !!! NEEDS TO BE CONVERTED
|
# PIPE Function
|
||||||
|
#
|
||||||
|
# Copies data from one file handle to another, optionally compressing or decompressing the data in stream.
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
sub file_copy
|
sub pipe()
|
||||||
|
{
|
||||||
|
my $self = shift;
|
||||||
|
my $hIn = shift;
|
||||||
|
my $hOut = shift;
|
||||||
|
my $bCompress = shift;
|
||||||
|
my $bUncompress = shift;
|
||||||
|
|
||||||
|
# If compression is requested and the file is not already compressed
|
||||||
|
if (defined($bCompress) && $bCompress)
|
||||||
|
{
|
||||||
|
if (!gzip($hIn => $hOut))
|
||||||
|
{
|
||||||
|
confess $GzipError;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# If no compression is requested and the file is already compressed
|
||||||
|
elsif (defined($bUncompress) && $bUncompress)
|
||||||
|
{
|
||||||
|
if (!gunzip($hIn => $hOut))
|
||||||
|
{
|
||||||
|
confess $GunzipError;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# Else it's a straight copy
|
||||||
|
else
|
||||||
|
{
|
||||||
|
my $strBuffer;
|
||||||
|
my $iResultRead;
|
||||||
|
my $iResultWrite;
|
||||||
|
|
||||||
|
# Read from the input handle
|
||||||
|
while (($iResultRead = sysread($hIn, $strBuffer, BLOCK_SIZE)) != 0)
|
||||||
|
{
|
||||||
|
if (!defined($iResultRead))
|
||||||
|
{
|
||||||
|
confess $!;
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
# Write to the output handle
|
||||||
|
$iResultWrite = syswrite($hOut, $strBuffer, $iResultRead);
|
||||||
|
|
||||||
|
if (!defined($iResultWrite) || $iResultWrite != $iResultRead)
|
||||||
|
{
|
||||||
|
confess $!;
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
####################################################################################################################################
|
||||||
|
# COPY
|
||||||
|
#
|
||||||
|
# Copies a file from one location to another:
|
||||||
|
#
|
||||||
|
# * source and destination can be local or remote
|
||||||
|
# * wire and output compression/decompression are supported
|
||||||
|
# * intermediate temp files are used to prevent partial copies
|
||||||
|
# * modification time and permissions can be set on destination file
|
||||||
|
# * destination path can optionally be created
|
||||||
|
####################################################################################################################################
|
||||||
|
sub copy
|
||||||
{
|
{
|
||||||
my $self = shift;
|
my $self = shift;
|
||||||
my $strSourcePathType = shift;
|
my $strSourcePathType = shift;
|
||||||
my $strSourceFile = shift;
|
my $strSourceFile = shift;
|
||||||
my $strDestinationPathType = shift;
|
my $strDestinationPathType = shift;
|
||||||
my $strDestinationFile = shift;
|
my $strDestinationFile = shift;
|
||||||
my $bNoCompressionOverride = shift;
|
my $bIgnoreMissingSource = shift;
|
||||||
|
my $bCompress = shift;
|
||||||
|
my $bPathCreate = shift;
|
||||||
my $lModificationTime = shift;
|
my $lModificationTime = shift;
|
||||||
my $strPermission = shift;
|
my $strPermission = shift;
|
||||||
my $bPathCreate = shift;
|
|
||||||
my $bConfessCopyError = shift;
|
|
||||||
|
|
||||||
# if bPathCreate is not defined, default to true
|
# Set defaults
|
||||||
$bPathCreate = defined($bPathCreate) ? $bPathCreate : true;
|
$bCompress = defined($bCompress) ? $bCompress : defined($self->{bNoCompression}) ? !$self->{bNoCompression} : true;
|
||||||
$bConfessCopyError = defined($bConfessCopyError) ? $bConfessCopyError : true;
|
$bIgnoreMissingSource = defined($bIgnoreMissingSource) ? $bIgnoreMissingSource : false;
|
||||||
|
$bPathCreate = defined($bPathCreate) ? $bPathCreate : false;
|
||||||
|
|
||||||
&log(TRACE, "file_copy: ${strSourcePathType}: " . (defined($strSourceFile) ? ":${strSourceFile}" : "") .
|
# Set working variables
|
||||||
" to ${strDestinationPathType}" . (defined($strDestinationFile) ? ":${strDestinationFile}" : ""));
|
my $strErrorPrefix = "File->copy";
|
||||||
|
my $bSourceRemote = $self->is_remote($strSourcePathType);
|
||||||
|
my $bDestinationRemote = $self->is_remote($strDestinationPathType);
|
||||||
|
my $strSourceOp = $self->path_get($strSourcePathType, $strSourceFile);
|
||||||
|
my $strDestinationOp = $self->path_get($strDestinationPathType, $strDestinationFile);
|
||||||
|
my $strDestinationTmpOp = $self->path_get($strDestinationPathType, $strDestinationFile, true);
|
||||||
|
my $strError;
|
||||||
|
|
||||||
# Modification time and permissions cannot be set remotely
|
# Output trace info
|
||||||
if ((defined($lModificationTime) || defined($strPermission)) && $self->is_remote($strDestinationPathType))
|
&log(TRACE, "${strErrorPrefix}:" . ($bSourceRemote ? " remote" : " local") . " ${strSourcePathType}:${strSourceFile}" .
|
||||||
|
" to " . ($bDestinationRemote ? " remote" : " local") . " ${strDestinationPathType}:${strDestinationFile}");
|
||||||
|
|
||||||
|
# If source or destination are remote
|
||||||
|
if ($bSourceRemote || $bDestinationRemote)
|
||||||
{
|
{
|
||||||
confess &log(ASSERT, "modification time and permissions cannot be set on remote destination file");
|
# Get the ssh connection
|
||||||
|
my $oSSH;
|
||||||
|
|
||||||
|
# If source is local and destination is remote then use the destination connection
|
||||||
|
if (!$bSourceRemote && $bDestinationRemote)
|
||||||
|
{
|
||||||
|
$oSSH = $self->remote_get($strDestinationPathType);
|
||||||
|
}
|
||||||
|
# Else source connection is always used (because if both are remote they must be the same remote)
|
||||||
|
else
|
||||||
|
{
|
||||||
|
$oSSH = $self->remote_get($strSourcePathType);
|
||||||
}
|
}
|
||||||
|
|
||||||
# Generate source, destination and tmp filenames
|
# Build the command and open the local file
|
||||||
my $strSource = $self->path_get($strSourcePathType, $strSourceFile);
|
|
||||||
my $strDestination = $self->path_get($strDestinationPathType, $strDestinationFile);
|
|
||||||
my $strDestinationTmp = $self->path_get($strDestinationPathType, $strDestinationFile, true);
|
|
||||||
|
|
||||||
# Is this already a compressed file?
|
|
||||||
my $bAlreadyCompressed = $strSource =~ "^.*\.$self->{strCompressExtension}\$";
|
|
||||||
|
|
||||||
if ($bAlreadyCompressed && $strDestination !~ "^.*\.$self->{strCompressExtension}\$")
|
|
||||||
{
|
|
||||||
$strDestination .= ".$self->{strCompressExtension}";
|
|
||||||
}
|
|
||||||
|
|
||||||
# Does the file need compression?
|
|
||||||
my $bCompress = !((defined($bNoCompressionOverride) && $bNoCompressionOverride) ||
|
|
||||||
(!defined($bNoCompressionOverride) && $self->{bNoCompression}));
|
|
||||||
|
|
||||||
# If the destination path is backup and does not exist, create it
|
|
||||||
if ($bPathCreate && $self->path_type_get($strDestinationPathType) eq PATH_BACKUP)
|
|
||||||
{
|
|
||||||
$self->path_create(PATH_BACKUP_ABSOLUTE, dirname($strDestination));
|
|
||||||
}
|
|
||||||
|
|
||||||
# Generate the command string depending on compression/decompression/cat
|
|
||||||
my $strCommand = $self->{strCommandCat};
|
|
||||||
|
|
||||||
if (!$bAlreadyCompressed && $bCompress)
|
|
||||||
{
|
|
||||||
$strCommand = $self->{strCommandCompress};
|
|
||||||
$strDestination .= ".gz";
|
|
||||||
}
|
|
||||||
elsif ($bAlreadyCompressed && !$bCompress)
|
|
||||||
{
|
|
||||||
$strCommand = $self->{strCommandDecompress};
|
|
||||||
$strDestination = substr($strDestination, 0, length($strDestination) - length($self->{strCompressExtension}) - 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
$strCommand =~ s/\%file\%/${strSource}/g;
|
|
||||||
|
|
||||||
# If this command is remote on only one side
|
|
||||||
if ($self->is_remote($strSourcePathType) && !$self->is_remote($strDestinationPathType) ||
|
|
||||||
!$self->is_remote($strSourcePathType) && $self->is_remote($strDestinationPathType))
|
|
||||||
{
|
|
||||||
# Else if the source is remote
|
|
||||||
if ($self->is_remote($strSourcePathType))
|
|
||||||
{
|
|
||||||
&log(TRACE, "file_copy: remote ${strSource} to local ${strDestination}");
|
|
||||||
|
|
||||||
# Open the destination file for writing (will be streamed from the ssh session)
|
|
||||||
my $hFile;
|
my $hFile;
|
||||||
open($hFile, ">", $strDestinationTmp) or confess &log(ERROR, "cannot open ${strDestination}");
|
my $strCommand;
|
||||||
|
|
||||||
# Execute the command through ssh
|
# If source is remote and destination is local
|
||||||
my $oSSH = $self->remote_get($strSourcePathType);
|
if ($bSourceRemote && !$bDestinationRemote)
|
||||||
|
|
||||||
unless ($oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file"), stdout_fh => $hFile}, $strCommand))
|
|
||||||
{
|
{
|
||||||
close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}");
|
# Build the command string
|
||||||
|
$strCommand = $self->{strCommand} .
|
||||||
|
" --compress copy_in ${strSourceOp}";
|
||||||
|
|
||||||
my $strResult = "unable to execute ssh '${strCommand}': " . $self->error_get();
|
open($hFile, ">", $strDestinationTmpOp)
|
||||||
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
|
or confess &log(ERROR, "cannot open ${strDestinationTmpOp}: " . $!);
|
||||||
|
}
|
||||||
|
# Else if source is local and destination is remote
|
||||||
|
elsif (!$bSourceRemote && $bDestinationRemote)
|
||||||
|
{
|
||||||
|
# !!! SOURCE LOCAL DESTINATION REMOTE COPY NOT YET IMPLEMENTED
|
||||||
|
return false;
|
||||||
|
|
||||||
|
# Build the command string
|
||||||
|
$strCommand = $self->{strCommand} .
|
||||||
|
" --compress copy_out ${strDestinationOp}";
|
||||||
|
|
||||||
|
# Open source file for reading
|
||||||
|
open($hFile, "<", $strSourceOp)
|
||||||
|
or confess &log(ERROR, "cannot open ${strSourceOp}: " . $!);
|
||||||
|
}
|
||||||
|
# Else source and destination are remote
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if ($self->path_type_get($strSourcePathType) ne $self->path_type_get($strDestinationPathType))
|
||||||
|
{
|
||||||
|
confess &log(ASSERT, "remote source and destination not supported");
|
||||||
}
|
}
|
||||||
|
|
||||||
# Close the destination file handle
|
# !!! MULTIPLE REMOTE COPY NOT YET IMPLEMENTED
|
||||||
close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}");
|
return false;
|
||||||
}
|
|
||||||
# Else if the destination is remote
|
|
||||||
elsif ($self->is_remote($strDestinationPathType))
|
|
||||||
{
|
|
||||||
&log(TRACE, "file_copy: local ${strSource} ($strCommand) to remote ${strDestination}");
|
|
||||||
|
|
||||||
if (defined($self->path_get(PATH_LOCK_ERR, "file")))
|
|
||||||
{
|
|
||||||
$strCommand .= " 2> " . $self->path_get(PATH_LOCK_ERR, "file");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Open the input command as a stream
|
# Execute the ssh command
|
||||||
my $hOut;
|
my ($hIn, $hOut, $hErr, $pId) = $oSSH->open3($strCommand)
|
||||||
my $pId = open3(undef, $hOut, undef, $strCommand) or confess(ERROR, "unable to execute '${strCommand}'");
|
or confess &log("unable to execute ssh '${strCommand}': " . $self->error_get());
|
||||||
|
|
||||||
# Execute the command though ssh
|
# If source is remote and destination is local
|
||||||
my $oSSH = $self->remote_get($strDestinationPathType);
|
if ($bSourceRemote && !$bDestinationRemote)
|
||||||
$oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file"), stdin_fh => $hOut}, "cat > ${strDestinationTmp}") or confess &log(ERROR, "unable to execute ssh 'cat'");
|
{
|
||||||
|
$self->pipe($hOut, $hFile, $bCompress, !$bCompress);
|
||||||
|
}
|
||||||
|
# Else if source is local and destination is remote
|
||||||
|
elsif (!$bSourceRemote && $bDestinationRemote)
|
||||||
|
{
|
||||||
|
$self->pipe($hFile, $hIn, $bCompress, !$bCompress);
|
||||||
|
}
|
||||||
|
|
||||||
# Wait for the stream process to finish
|
# Read STDERR into a string
|
||||||
|
my $strError = "";
|
||||||
|
|
||||||
|
open my ($hErrOut), '>', $strError;
|
||||||
|
$self->pipe($hErr, $hErrOut);
|
||||||
|
close($hErrOut);
|
||||||
|
|
||||||
|
# Read STDOUT into a string
|
||||||
|
my $strOutput = "";
|
||||||
|
|
||||||
|
open my ($hOutString), '>', $strOutput;
|
||||||
|
$self->pipe($hOut, $hOutString);
|
||||||
|
close($hOutString);
|
||||||
|
|
||||||
|
# Wait for the process to finish and report any errors
|
||||||
waitpid($pId, 0);
|
waitpid($pId, 0);
|
||||||
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
|
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
|
||||||
|
|
||||||
if ($iExitStatus != 0)
|
if ($iExitStatus != 0)
|
||||||
{
|
{
|
||||||
my $strResult = "command '${strCommand}' returned " . $iExitStatus . ": " . $self->error_get();
|
confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus . ": " . $strError);
|
||||||
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
# If the source and destination are both remote but not the same remote
|
|
||||||
elsif ($self->is_remote($strSourcePathType) && $self->is_remote($strDestinationPathType) &&
|
|
||||||
$self->path_type_get($strSourcePathType) ne $self->path_type_get($strDestinationPathType))
|
|
||||||
{
|
|
||||||
&log(TRACE, "file_copy: remote ${strSource} to remote ${strDestination}");
|
|
||||||
confess &log(ASSERT, "remote source and destination not supported");
|
|
||||||
}
|
|
||||||
# Else this is a local command or remote where both sides are the same remote
|
|
||||||
else
|
|
||||||
{
|
|
||||||
# Complete the command by redirecting to the destination tmp file
|
|
||||||
$strCommand .= " > ${strDestinationTmp}";
|
|
||||||
|
|
||||||
if ($self->is_remote($strSourcePathType))
|
# Close the destination file handle
|
||||||
|
if (defined($hFile))
|
||||||
{
|
{
|
||||||
&log(TRACE, "file_copy: remote ${strSourcePathType} '${strCommand}'");
|
close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
|
||||||
|
|
||||||
my $oSSH = $self->remote_get($strSourcePathType);
|
|
||||||
$oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file")}, $strCommand);
|
|
||||||
|
|
||||||
if ($oSSH->error)
|
|
||||||
{
|
|
||||||
my $strResult = "unable to execute copy ${strCommand}: " . $oSSH->error;
|
|
||||||
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
&log(TRACE, "file_copy: local '${strCommand}'");
|
# !!! LOCAL COPY NOT YET IMPLEMENTED
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (defined($self->path_get(PATH_LOCK_ERR, "file")))
|
if (!$bDestinationRemote)
|
||||||
{
|
{
|
||||||
$strCommand .= " 2> " . $self->path_get(PATH_LOCK_ERR, "file");
|
# Set the file permission if required
|
||||||
}
|
|
||||||
|
|
||||||
unless(system($strCommand) == 0)
|
|
||||||
{
|
|
||||||
my $strResult = "unable to copy local ${strSource} to local ${strDestinationTmp}";
|
|
||||||
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Set the file permission if required (this only works locally for now)
|
|
||||||
if (defined($strPermission))
|
if (defined($strPermission))
|
||||||
{
|
{
|
||||||
&log(TRACE, "file_copy: chmod ${strPermission}");
|
system("chmod ${strPermission} ${strDestinationTmpOp}") == 0
|
||||||
|
or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmpOp}");
|
||||||
system("chmod ${strPermission} ${strDestinationTmp}") == 0
|
|
||||||
or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmp}");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Set the file modification time if required (this only works locally for now)
|
# Set the file modification time if required (this only works locally for now)
|
||||||
if (defined($lModificationTime))
|
if (defined($lModificationTime))
|
||||||
{
|
{
|
||||||
&log(TRACE, "file_copy: time ${lModificationTime}");
|
utime($lModificationTime, $lModificationTime, $strDestinationTmpOp)
|
||||||
|
or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}");
|
||||||
utime($lModificationTime, $lModificationTime, $strDestinationTmp)
|
|
||||||
or confess &log(ERROR, "unable to set time for local ${strDestinationTmp}");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Move the file from tmp to final destination
|
# Move the file from tmp to final destination
|
||||||
$self->file_move($self->path_type_get($strSourcePathType) . ":absolute", $strDestinationTmp,
|
$self->move($self->path_type_get($strDestinationPathType) . ":absolute", $strDestinationTmpOp,
|
||||||
$self->path_type_get($strDestinationPathType) . ":absolute", $strDestination, $bPathCreate);
|
$self->path_type_get($strDestinationPathType) . ":absolute", $strDestinationOp, $bPathCreate);
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -34,7 +34,6 @@ sub BackRestFileTest
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Setup test paths
|
# Setup test paths
|
||||||
my $strLockPath = dirname(abs_path($0)) . "/lock";
|
|
||||||
my $strTestPath = dirname(abs_path($0)) . "/test";
|
my $strTestPath = dirname(abs_path($0)) . "/test";
|
||||||
my $iRun;
|
my $iRun;
|
||||||
|
|
||||||
@ -739,19 +738,39 @@ sub BackRestFileTest
|
|||||||
{
|
{
|
||||||
$iRun = 0;
|
$iRun = 0;
|
||||||
|
|
||||||
system("rm -rf lock");
|
# my $strLockPath = dirname(abs_path($0)) . "/test/lock";
|
||||||
system("mkdir -p lock") == 0 or confess "Unable to create lock directory";
|
#
|
||||||
|
#
|
||||||
|
# my $oFile = pg_backrest_file->new
|
||||||
|
# (
|
||||||
|
# strStanza => "db",
|
||||||
|
# bNoCompression => false,
|
||||||
|
# strBackupClusterPath => undef,
|
||||||
|
# strBackupPath => ${strTestPath},
|
||||||
|
# strBackupHost => $strHost,
|
||||||
|
# strBackupUser => $strUser,
|
||||||
|
# strDbHost => undef,
|
||||||
|
# strDbUser => undef,
|
||||||
|
# strLockPath => $strLockPath
|
||||||
|
# );
|
||||||
|
#
|
||||||
|
# my $strSourceFile = "${strTestPath}/backup/test.txt";
|
||||||
|
# my $strDestinationFile = "${strTestPath}/db/test.txt";
|
||||||
|
#
|
||||||
|
# system("echo 'TESTDATA' > ${strSourceFile}");
|
||||||
|
#
|
||||||
|
# $oFile->copy(PATH_BACKUP_ABSOLUTE, $strSourceFile, PATH_DB_ABSOLUTE, $strDestinationFile, undef, false);
|
||||||
|
|
||||||
for (my $bBackupRemote = 0; $bBackupRemote <= 1; $bBackupRemote++)
|
for (my $bBackupRemote = 0; $bBackupRemote <= 1; $bBackupRemote++)
|
||||||
{
|
{
|
||||||
my $strBackupHost = $bBackupRemote ? "127.0.0.1" : undef;
|
|
||||||
my $strBackupUser = $bBackupRemote ? "dsteele" : undef;
|
|
||||||
|
|
||||||
# Loop through source compression
|
# Loop through source compression
|
||||||
for (my $bDbRemote = 0; $bDbRemote <= 1; $bDbRemote++)
|
for (my $bDbRemote = 0; $bDbRemote <= 1; $bDbRemote++)
|
||||||
{
|
{
|
||||||
my $strDbHost = $bDbRemote ? "127.0.0.1" : undef;
|
# Backup and db cannot both be remote
|
||||||
my $strDbUser = $bDbRemote ? "dsteele" : undef;
|
if ($bBackupRemote && $bDbRemote)
|
||||||
|
{
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
# Loop through destination compression
|
# Loop through destination compression
|
||||||
for (my $bDestinationCompressed = 0; $bDestinationCompressed <= 1; $bDestinationCompressed++)
|
for (my $bDestinationCompressed = 0; $bDestinationCompressed <= 1; $bDestinationCompressed++)
|
||||||
@ -759,45 +778,42 @@ sub BackRestFileTest
|
|||||||
my $oFile = pg_backrest_file->new
|
my $oFile = pg_backrest_file->new
|
||||||
(
|
(
|
||||||
strStanza => "db",
|
strStanza => "db",
|
||||||
|
strCommand => $strCommand,
|
||||||
bNoCompression => !$bDestinationCompressed,
|
bNoCompression => !$bDestinationCompressed,
|
||||||
strBackupClusterPath => undef,
|
strBackupClusterPath => undef,
|
||||||
strBackupPath => ${strTestPath},
|
strBackupPath => ${strTestPath},
|
||||||
strBackupHost => $strBackupHost,
|
strBackupHost => $bBackupRemote ? $strHost : undef,
|
||||||
strBackupUser => $strBackupUser,
|
strBackupUser => $bBackupRemote ? $strUser : undef,
|
||||||
strDbHost => $strDbHost,
|
strDbHost => $bDbRemote ? $strHost : undef,
|
||||||
strDbUser => $strDbUser,
|
strDbUser => $bDbRemote ? $strUser : undef
|
||||||
strCommandCompress => "gzip --stdout %file%",
|
|
||||||
strCommandDecompress => "gzip -dc %file%",
|
|
||||||
strLockPath => dirname($0) . "/test/lock"
|
|
||||||
);
|
);
|
||||||
|
|
||||||
for (my $bSourceCompressed = 0; $bSourceCompressed <= 1; $bSourceCompressed++)
|
for (my $bSourceCompressed = 0; $bSourceCompressed <= 1; $bSourceCompressed++)
|
||||||
{
|
{
|
||||||
for (my $bSourcePathType = 0; $bSourcePathType <= 1; $bSourcePathType++)
|
for (my $bSourcePathType = 0; $bSourcePathType <= 1; $bSourcePathType++)
|
||||||
{
|
{
|
||||||
my $strSourcePath = $bSourcePathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
|
my $strSourcePathType = $bSourcePathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
|
||||||
|
my $strSourcePath = $bSourcePathType ? "db" : "backup";
|
||||||
|
|
||||||
for (my $bDestinationPathType = 0; $bDestinationPathType <= 1; $bDestinationPathType++)
|
for (my $bDestinationPathType = 0; $bDestinationPathType <= 1; $bDestinationPathType++)
|
||||||
{
|
{
|
||||||
my $strDestinationPath = $bDestinationPathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
|
my $strDestinationPathType = $bDestinationPathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
|
||||||
|
my $strDestinationPath = $bDestinationPathType ? "db" : "backup";
|
||||||
|
|
||||||
for (my $bError = 0; $bError <= 1; $bError++)
|
|
||||||
{
|
|
||||||
for (my $bConfessError = 0; $bConfessError <= 1; $bConfessError++)
|
|
||||||
{
|
|
||||||
$iRun++;
|
$iRun++;
|
||||||
|
|
||||||
print "run ${iRun} - " .
|
&log(INFO, "run ${iRun} - " .
|
||||||
"srcpth ${strSourcePath}, bkprmt $bBackupRemote, srccmp $bSourceCompressed, " .
|
"srcpth ${strSourcePath}, bkprmt $bBackupRemote, srccmp $bSourceCompressed, " .
|
||||||
"dstpth ${strDestinationPath}, dbrmt $bDbRemote, dstcmp $bDestinationCompressed, " .
|
"dstpth ${strDestinationPath}, dbrmt $bDbRemote, dstcmp $bDestinationCompressed");
|
||||||
"error $bError, confess_error $bConfessError\n";
|
|
||||||
|
|
||||||
# Drop the old test directory and create a new one
|
# Drop the old test directory and create a new one
|
||||||
system("rm -rf test");
|
system("rm -rf test");
|
||||||
system("mkdir -p test/lock") == 0 or confess "Unable to create the test directory";
|
system("mkdir -p test/lock") == 0 or confess "Unable to create test/lock directory";
|
||||||
|
system("mkdir -p test/backup") == 0 or confess "Unable to create test/backup directory";
|
||||||
|
system("mkdir -p test/db") == 0 or confess "Unable to create test/db directory";
|
||||||
|
|
||||||
my $strSourceFile = "${strTestPath}/test-source.txt";
|
my $strSourceFile = "${strTestPath}/${strSourcePath}/test-source.txt";
|
||||||
my $strDestinationFile = "${strTestPath}/test-destination.txt";
|
my $strDestinationFile = "${strTestPath}/${strDestinationPath}/test-destination.txt";
|
||||||
|
|
||||||
# Create the compressed or uncompressed test file
|
# Create the compressed or uncompressed test file
|
||||||
if ($bSourceCompressed)
|
if ($bSourceCompressed)
|
||||||
@ -811,97 +827,95 @@ sub BackRestFileTest
|
|||||||
}
|
}
|
||||||
# Create the file object based on current values
|
# Create the file object based on current values
|
||||||
|
|
||||||
if ($bError)
|
|
||||||
{
|
|
||||||
$strSourceFile .= ".error";
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run file copy in an eval block because some errors are expected
|
# Run file copy in an eval block because some errors are expected
|
||||||
my $bReturn;
|
my $bReturn;
|
||||||
|
|
||||||
eval
|
eval
|
||||||
{
|
{
|
||||||
$bReturn = $oFile->file_copy($strSourcePath, $strSourceFile,
|
$bReturn = $oFile->copy($strSourcePathType, $strSourceFile,
|
||||||
$strDestinationPath, $strDestinationFile,
|
$strDestinationPathType, $strDestinationFile);
|
||||||
undef, undef, undef, undef,
|
|
||||||
$bConfessError);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# Check for errors after copy
|
# Check for errors after copy
|
||||||
if ($@)
|
if ($@)
|
||||||
{
|
{
|
||||||
# Different remote and destination with different path types should error
|
# Different remote and destination with different path types should error
|
||||||
if ($bBackupRemote && $bDbRemote && ($strSourcePath ne $strDestinationPath))
|
if (($bBackupRemote || $bDbRemote) && ($strSourcePathType ne $strDestinationPathType))
|
||||||
{
|
{
|
||||||
print " different source and remote for same path not supported\n";
|
print " different source and remote for same path not supported\n";
|
||||||
next;
|
next;
|
||||||
}
|
}
|
||||||
# If the error was intentional, then also continue
|
# If the error was intentional, then also continue
|
||||||
elsif ($bError)
|
# elsif ($bError)
|
||||||
{
|
# {
|
||||||
my $strError = $oFile->error_get();
|
# my $strError = $oFile->error_get();
|
||||||
|
#
|
||||||
if (!defined($strError) || ($strError eq ''))
|
# if (!defined($strError) || ($strError eq ''))
|
||||||
{
|
# {
|
||||||
confess 'no error message returned';
|
# confess 'no error message returned';
|
||||||
}
|
# }
|
||||||
|
#
|
||||||
print " error raised: ${strError}\n";
|
# print " error raised: ${strError}\n";
|
||||||
next;
|
# next;
|
||||||
}
|
# }
|
||||||
# Else this is an unexpected error
|
# Else this is an unexpected error
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
confess $@;
|
confess $@;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
elsif ($bError)
|
# elsif ($bError)
|
||||||
{
|
# {
|
||||||
if ($bConfessError)
|
# if ($bConfessError)
|
||||||
{
|
# {
|
||||||
confess "Value was returned instead of exception thrown when confess error is true";
|
# confess "Value was returned instead of exception thrown when confess error is true";
|
||||||
}
|
# }
|
||||||
else
|
# else
|
||||||
{
|
# {
|
||||||
if ($bReturn)
|
# if ($bReturn)
|
||||||
{
|
# {
|
||||||
confess "true was returned when an error was generated";
|
# confess "true was returned when an error was generated";
|
||||||
}
|
# }
|
||||||
else
|
# else
|
||||||
{
|
# {
|
||||||
my $strError = $oFile->error_get();
|
# my $strError = $oFile->error_get();
|
||||||
|
#
|
||||||
if (!defined($strError) || ($strError eq ''))
|
# if (!defined($strError) || ($strError eq ''))
|
||||||
{
|
# {
|
||||||
confess 'no error message returned';
|
# confess 'no error message returned';
|
||||||
}
|
# }
|
||||||
|
#
|
||||||
print " error returned: ${strError}\n";
|
# print " error returned: ${strError}\n";
|
||||||
next;
|
# next;
|
||||||
}
|
# }
|
||||||
}
|
# }
|
||||||
}
|
# }
|
||||||
else
|
# else
|
||||||
{
|
# {
|
||||||
if (!$bReturn)
|
# if (!$bReturn)
|
||||||
{
|
# {
|
||||||
confess "error was returned when no error generated";
|
# confess "error was returned when no error generated";
|
||||||
}
|
# }
|
||||||
|
#
|
||||||
print " true was returned\n";
|
# print " true was returned\n";
|
||||||
}
|
# }
|
||||||
|
|
||||||
# Check for errors after copy
|
# Check for errors after copy
|
||||||
if ($bDestinationCompressed)
|
# if ($bDestinationCompressed)
|
||||||
{
|
# {
|
||||||
$strDestinationFile .= ".gz";
|
# $strDestinationFile .= ".gz";
|
||||||
}
|
# }
|
||||||
|
|
||||||
|
if ($bReturn)
|
||||||
|
{
|
||||||
unless (-e $strDestinationFile)
|
unless (-e $strDestinationFile)
|
||||||
{
|
{
|
||||||
confess "could not find destination file ${strDestinationFile}";
|
confess "could not find destination file ${strDestinationFile}";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
&log(INFO, "Not yet implemented");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user