db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-10-30 23:37:45 +02:00
Code Issues Releases Activity

In-stream compression now working in copy.

Browse Source
This commit is contained in:
David Steele
2014-06-15 15:56:45 -04:00
parent f51c06119c
commit adf654edb3
4 changed files with 317 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bin/pg_backrest_remote.pl
View File

@@ -121,7 +121,7 @@ while ($strCommand ne OP_EXIT)
}
}
};
if ($@)
{
$oRemote->error_write($@);

249
lib/BackRest/File.pm
View File

@@ -27,12 +27,12 @@ use BackRest::Remote;
use Exporter qw(import);
our @EXPORT = qw(PATH_ABSOLUTE PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_ABSOLUTE
PATH_BACKUP_CLUSTERPATH_BACKUP_TMP PATH_BACKUP_ARCHIVE
COMMAND_ERR_FILE_MISSING COMMAND_ERR_FILE_READ COMMAND_ERR_FILE_MOVE COMMAND_ERR_FILE_TYPE
COMMAND_ERR_LINK_READ COMMAND_ERR_PATH_MISSING COMMAND_ERR_PATH_CREATE COMMAND_ERR_PARAM
PIPE_STDIN PIPE_STDOUT PIPE_STDERR
OP_FILE_LIST OP_FILE_EXISTS OP_FILE_HASH OP_FILE_REMOVE OP_FILE_MANIFEST OP_FILE_COMPRESS
OP_FILE_MOVE OP_FILE_COPY_OUT OP_FILE_COPY_IN OP_FILE_PATH_CREATE);
@@ -603,11 +603,11 @@ sub move
# {
# my $self = shift;
# my $hOut = shift;
#
#
# my $strBuffer;
# my $hString = IO::String->new($strBuffer);
# $self->pipe($hOut, $hString);
#
#
# return $strBuffer;
# }
@@ -623,7 +623,7 @@ sub move
# my $hOut = shift;
# my $bCompress = shift;
# my $bUncompress = shift;
#
#
# # If compression is requested and the file is not already compressed
# if (defined($bCompress) && $bCompress)
# {
@@ -646,7 +646,7 @@ sub move
# my $strBuffer;
# my $iResultRead;
# my $iResultWrite;
#
#
# # Read from the input handle
# while (($iResultRead = sysread($hIn, $strBuffer, BLOCK_SIZE)) != 0)
# {
@@ -659,7 +659,7 @@ sub move
# {
# # Write to the output handle
# $iResultWrite = syswrite($hOut, $strBuffer, $iResultRead);
#
#
# if (!defined($iResultWrite) || $iResultWrite != $iResultRead)
# {
# confess $!;
@@ -681,17 +681,17 @@ sub move
# my $hIn = shift;
# my $hOut = shift;
# my $hErr = shift;
#
#
# # Close hIn
# close($hIn);
#
#
# # Read STDERR into a string
# my $strError = defined($hErr) ? $self->pipe_to_string($hErr) : "[unknown]";
#
#
# # Wait for the process to finish and report any errors
# waitpid($pId, 0);
# my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
#
#
# if ($iExitStatus != 0)
# {
# confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus . ": " .
@@ -699,6 +699,72 @@ sub move
# }
# }
####################################################################################################################################
# EXISTS - Checks for the existence of a file, but does not imply that the file is readable/writeable.
#
# Return: true if file exists, false otherwise
####################################################################################################################################
sub exists
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
# Set operation variables
my $strPathOp = $self->path_get($strPathType, $strPath);
# Set operation and debug strings
my $strOperation = OP_FILE_EXISTS;
my $strDebug = "${strPathType}:${strPathOp}";
# Run remotely
if ($self->is_remote($strPathType))
{
# Build param hash
my %oParamHash;
$oParamHash{path} = ${strPathOp};
# Build debug string
$strDebug = "${strOperation}: remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . "): " . $strDebug;
&log(DEBUG, $strDebug);
# Execute the command
return $self->{oRemote}->command_execute($strOperation, \%oParamHash, true, $strDebug) eq "Y";
}
# Run locally
else
{
# Build debug string
$strDebug = "${strOperation}: local: " . $strDebug;
&log(DEBUG, ${strDebug});
# Stat the file/path to determine if it exists
my $oStat = lstat($strPathOp);
# Evaluate error
if (!defined($oStat))
{
# If the error is not entry missing, then throw error
if (!$!{ENOENT})
{
if ($strPathType eq PATH_ABSOLUTE)
{
confess &log(ERROR, $!, COMMAND_ERR_FILE_READ);
}
else
{
confess &log(ERROR, "${strDebug}: " . $!, COMMAND_ERR_FILE_READ);
}
}
return false;
}
}
return true;
}
####################################################################################################################################
# COPY
#
@@ -724,12 +790,12 @@ sub copy
my $strPermission = shift;
# Set defaults
my $bIsCompressed = false;
$bCompress = defined($bCompress) ? $bCompress : defined($self->{bCompress}) ? $self->{bCompress} : true;
$bIgnoreMissingSource = defined($bIgnoreMissingSource) ? $bIgnoreMissingSource : false;
$bPathCreate = defined($bPathCreate) ? $bPathCreate : false;
# Set working variables
my $strErrorPrefix = "File->copy";
my $bSourceRemote = $self->is_remote($strSourcePathType) || $strSourcePathType eq PIPE_STDIN;
my $bDestinationRemote = $self->is_remote($strDestinationPathType) || $strDestinationPathType eq PIPE_STDOUT;
my $strSourceOp = $strSourcePathType eq PIPE_STDIN ?
@@ -739,22 +805,22 @@ sub copy
my $strDestinationTmpOp = $strDestinationPathType eq PIPE_STDOUT ?
undef : $self->path_get($strDestinationPathType, $strDestinationFile, true);
# Determine if the file needs compression extension
if ($bCompress && $strDestinationOp !~ "^.*\.$self->{strCompressExtension}\$")
{
$strDestinationOp .= "." . $self->{strCompressExtension};
}
# Output trace info
# &log(TRACE, "${strErrorPrefix}:" . ($bSourceRemote ? " remote" : " local") . " ${strSourcePathType}:${strSourceFile}" .
# " to" . ($bDestinationRemote ? " remote" : " local") . " ${strDestinationPathType}:${strDestinationFile}" .
# ", compress = " . ($bCompress ? "true" : "false"));
# Set operation and debug string
my $strOperation = "File->unknown";
my $strDebug = ($bSourceRemote ? " remote" : " local") . " ${strSourcePathType}" .
(defined($strSourceFile) ? ":${strSourceFile}" : "") .
" to" . ($bDestinationRemote ? " remote" : " local") . " ${strDestinationPathType}" .
(defined($strDestinationFile) ? ":${strDestinationFile}" : "") .
", compress = " . ($bCompress ? "true" : "false");
# Open the source file
my $hSourceFile;
if (!$bSourceRemote)
{
# Determine if the file is compressed
$bIsCompressed = $strSourceOp !~ "^.*\.$self->{strCompressExtension}\$";
open($hSourceFile, "<", $strSourceOp)
or confess &log(ERROR, "cannot open ${strSourceOp}: " . $!);
}
@@ -764,17 +830,21 @@ sub copy
if (!$bDestinationRemote)
{
# Determine if the file needs compression extension
if ($bCompress && $strDestinationOp !~ "^.*\.$self->{strCompressExtension}\$")
{
$strDestinationOp .= "." . $self->{strCompressExtension};
}
open($hDestinationFile, ">", $strDestinationTmpOp)
or confess &log(ERROR, "cannot open ${strDestinationTmpOp}: " . $!);
}
# If source or destination are remote
if ($bSourceRemote || $bDestinationRemote)
{
# print "got outside\n";
# Build the command and open the local file
my $hFile;
my $strOperation;
my %oParamHash;
my $hIn,
my $hOut;
@@ -784,17 +854,26 @@ sub copy
if ($bSourceRemote && !$bDestinationRemote)
{
$hOut = $hDestinationFile;
$strOperation = OP_FILE_COPY_OUT;
$bCompress = $bCompress ? undef : $bCompress;
if ($strSourcePathType eq PIPE_STDIN)
{
$strRemote = 'in';
$hIn = *STDIN;
}
else
{
$strRemote = 'in';
$strOperation = OP_FILE_COPY_OUT;
$oParamHash{source_file} = ${strSourceOp};
if (defined($bCompress))
{
$oParamHash{compress} = $bCompress;
}
$hIn = $self->{oRemote}->{hOut};
}
}
@@ -802,7 +881,8 @@ sub copy
elsif (!$bSourceRemote && $bDestinationRemote)
{
$hIn = $hSourceFile;
$strOperation = OP_FILE_COPY_IN;
if ($strDestinationPathType eq PIPE_STDOUT)
{
$strRemote = 'out';
@@ -811,14 +891,19 @@ sub copy
else
{
$strRemote = 'out';
$strOperation = OP_FILE_COPY_IN;
$oParamHash{destination_file} = ${strDestinationOp};
$bCompress = $bCompress ? undef : $bCompress;
if (defined($bCompress))
{
$oParamHash{compress} = $bCompress;
}
$hOut = $self->{oRemote}->{hIn};
}
# Build debug string
# $strDebug = "${strOperation}: remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . "): " . $strDebug;
# &log(DEBUG, $strDebug);
$bCompress = true;
}
# Else source and destination are remote
else
@@ -832,28 +917,25 @@ sub copy
return false;
}
# If an operation is defined then write it
if (defined($strOperation))
{
# Trace command
&log(TRACE, "${strErrorPrefix} operation:" . $strOperation);
# Build debug string
$strDebug = "${strOperation}: " . (%oParamHash ? "remote (" .
$self->{oRemote}->command_param_string(\%oParamHash) . ") :" : "") . $strDebug;
&log(DEBUG, $strDebug);
# Execute the operation
# If an operation is defined then write it
if (%oParamHash)
{
$self->{oRemote}->command_write($strOperation, \%oParamHash);
}
# Transfer the file
# print "binary xfer start\n";
$self->{oRemote}->binary_xfer($hIn, $hOut, $strRemote);
# print "binary xfer stop\n";
if ($strSourcePathType ne PIPE_STDIN && $strDestinationPathType ne PIPE_STDOUT)
# Transfer the file
$self->{oRemote}->binary_xfer($hIn, $hOut, $strRemote, $bCompress);
# If this is the controlling process then wait for OK from remote
if (%oParamHash)
{
$self->{oRemote}->output_read(false, $strErrorPrefix);
$self->{oRemote}->output_read(false, $strDebug);
}
# Wait for process exit (and error)
# $self->wait_pid($pId, $strCommand, $hIn, $hOut, $hErr);
}
else
{
@@ -890,7 +972,6 @@ sub copy
or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}");
}
# Move the file from tmp to final destination
$self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, $bPathCreate);
}
@@ -1114,72 +1195,6 @@ sub list
return @stryFileList;
}
####################################################################################################################################
# EXISTS - Checks for the existence of a file, but does not imply that the file is readable/writeable.
#
# Return: true if file exists, false otherwise
####################################################################################################################################
sub exists
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
# Set operation variables
my $strPathOp = $self->path_get($strPathType, $strPath);
# Set operation and debug strings
my $strOperation = OP_FILE_EXISTS;
my $strDebug = "${strPathType}:${strPathOp}";
# Run remotely
if ($self->is_remote($strPathType))
{
# Build param hash
my %oParamHash;
$oParamHash{path} = ${strPathOp};
# Build debug string
$strDebug = "${strOperation}: remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . "): " . $strDebug;
&log(DEBUG, $strDebug);
# Execute the command
return $self->{oRemote}->command_execute($strOperation, \%oParamHash, true, $strDebug) eq "Y";
}
# Run locally
else
{
# Build debug string
$strDebug = "${strOperation}: local: " . $strDebug;
&log(DEBUG, ${strDebug});
# Stat the file/path to determine if it exists
my $oStat = lstat($strPathOp);
# Evaluate error
if (!defined($oStat))
{
# If the error is not entry missing, then throw error
if (!$!{ENOENT})
{
if ($strPathType eq PATH_ABSOLUTE)
{
confess &log(ERROR, $!, COMMAND_ERR_FILE_READ);
}
else
{
confess &log(ERROR, "${strDebug}: " . $!, COMMAND_ERR_FILE_READ);
}
}
return false;
}
}
return true;
}
####################################################################################################################################
# REMOVE
####################################################################################################################################

216
lib/BackRest/Remote.pm
View File

@@ -12,7 +12,9 @@ use Moose;
use Net::OpenSSH;
use File::Basename;
use POSIX ":sys_wait_h";
use IO::Compress::Gzip qw(gzip $GzipError);
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
use lib dirname($0) . "/../lib";
use BackRest::Exception;
use BackRest::Utility;
@@ -22,7 +24,7 @@ use BackRest::Utility;
####################################################################################################################################
use constant
{
DEFAULT_BLOCK_SIZE => 8192
DEFAULT_BLOCK_SIZE => 4
};
####################################################################################################################################
@@ -54,7 +56,7 @@ has iBlockSize => (is => 'bare', default => DEFAULT_BLOCK_SIZE); # Set block si
sub BUILD
{
my $self = shift;
$self->{strGreeting} .= " " . version_get();
if (defined($self->{strHost}))
@@ -95,7 +97,7 @@ sub BUILD
}
else
{
}
}
@@ -224,7 +226,7 @@ sub read_line
my $self = shift;
my $hIn = shift;
my $bError = shift;
my $strLine;
my $strChar;
my $iByteIn;
@@ -232,7 +234,7 @@ sub read_line
while (1)
{
$iByteIn = sysread($hIn, $strChar, 1);
if (!defined($iByteIn) || $iByteIn != 1)
{
$self->wait_pid();
@@ -241,10 +243,10 @@ sub read_line
{
return undef;
}
confess &log(ERROR, "unable to read 1 byte" . (defined($!) ? ": " . $! : ""));
}
if ($strChar eq "\n")
{
last;
@@ -252,7 +254,7 @@ sub read_line
$strLine .= $strChar;
}
return $strLine;
}
@@ -266,7 +268,7 @@ sub wait_pid
if (defined($self->{pId}) && waitpid($self->{pId}, WNOHANG) != 0)
{
my $strError = "no error on stderr";
if (!defined($self->{hErr}))
{
$strError = "no error captured because stderr is already closed";
@@ -280,7 +282,7 @@ sub wait_pid
$self->{hIn} = undef;
$self->{hOut} = undef;
$self->{hErr} = undef;
confess &log(ERROR, "remote process terminated: ${strError}");
}
}
@@ -301,30 +303,85 @@ sub binary_xfer
$strRemote = defined($strRemote) ? $strRemote : 'none';
my $iBlockSize = $self->{iBlockSize};
my $iHeaderSize = 12;
my $iBlockIn;
my $iBlockOut;
my $strBlockHeader;
my $strBlock;
my $oGzip;
my $hPipeIn;
my $hPipeOut;
my $pId;
if (!defined($hIn) || !defined($hOut))
{
confess &log(ASSERT, "hIn or hOut is not defined");
}
if ($strRemote eq "out")
{
pipe $hPipeOut, $hPipeIn;
# fork and exit the parent process
$pId = fork();
if (!$pId)
{
close($hPipeOut);
gzip($hIn => $hPipeIn)
or die confess &log(ERROR, "unable to compress: " . $GzipError);
close($hPipeIn);
exit 0;
}
close($hPipeIn);
$hIn = $hPipeOut;
}
elsif ($strRemote eq "in" && defined($bCompress) && !$bCompress)
{
pipe $hPipeOut, $hPipeIn;
# fork and exit the parent process
$pId = fork();
if (!$pId)
{
close($hPipeIn);
gunzip($hPipeOut => $hOut)
or die confess &log(ERROR, "unable to uncompress: " . $GunzipError);
close($hPipeOut);
exit 0;
}
# exit 0;
close($hPipeOut);
$hOut = $hPipeIn;
}
while (1)
{
if ($strRemote eq 'in')
{
$strBlockHeader = $self->read_line($hIn);
if ($strBlockHeader !~ /^block [0-9]+$/)
{
confess "unable to read block header ${strBlockHeader}";
}
$iBlockSize = trim(substr($strBlockHeader, index($strBlockHeader, " ") + 1));
# confess "found $iBlockSize to write";
if ($iBlockSize != 0)
{
$iBlockIn = sysread($hIn, $strBlock, $iBlockSize);
@@ -336,35 +393,73 @@ sub binary_xfer
}
else
{
#
# if ($strRemote eq 'in')
# {
# confess &log(ERROR, "block size $iBlockSize");
# }
$iBlockIn = 0;
}
}
else
{
if (!defined($bCompress))
$iBlockIn = sysread($hIn, $strBlock, $iBlockSize);
if (!defined($iBlockIn))
{
$iBlockIn = sysread($hIn, $strBlock, $iBlockSize);
if (!defined($iBlockIn))
{
confess "unable to read ${iBlockSize} bytes from remote: " . $!;
}
confess &log(ERROR, "unable to read");
}
}
# if (defined($bCompress))
# {
# if ($iBlockIn > 0)
# {
# my $iBlockInTmp = $iBlockIn;
#
# # print "${iBlockIn} bytes to compress";
#
# if ($bCompress)
# {
# $iBlockIn = $oGzip->syswrite($hIn, $iBlockInTmp);
#
# if (!defined($iBlockIn))
# {
# confess &log(ERROR, "unable to compress stream: " . $GunzipError)
# }
#
# if ($iBlockIn != $iBlockInTmp)
# {
# confess &log(ERROR, "unable to read ${iBlockSize} bytes");
# }
# }
# else
# {
# $iBlockIn = $oGzip->sysread($strBlock, 8192);
#
# if (!defined($iBlockIn))
# {
# confess &log(ERROR, "unable to read compressed stream: " . $GunzipError)
# }
# # $strBlock = $strBlockTmp;
# }
# }
# else
# {
# if ($bCompress)
# {
# $oGzip->flush()
# or confess &log(ERROR, "unable to flush compressed stream");
# $bDone = true;
# }
# }
#
# $iBlockIn = length($strBlock);
# }
if ($strRemote eq 'out')
{
$strBlockHeader = "block ${iBlockIn}\n";
# print "wrote block header: ${strBlockHeader}"; # REMOVE!
$iBlockOut = syswrite($hOut, $strBlockHeader);
if (!defined($iBlockOut) || $iBlockOut != length($strBlockHeader))
{
confess "unable to write block header";
@@ -382,12 +477,63 @@ sub binary_xfer
{
confess "unable to write ${iBlockIn} bytes" . (defined($!) ? ": " . $! : "");
}
# $strBlock = undef;
# if ($bDone && $strRemote eq 'out')
# {
# $strBlockHeader = "block 0\n";
#
# # print "wrote block header: ${strBlockHeader}"; # REMOVE!
#
# $iBlockOut = syswrite($hOut, $strBlockHeader);
#
# if (!defined($iBlockOut) || $iBlockOut != length($strBlockHeader))
# {
# confess "unable to write block header";
# }
# }
}
else
{
last;
# if ($strRemote eq 'in')
# {
# confess "got out\n";
# }
#
# if (defined($hPipeIn))
# {
# close($hPipeIn)
# };
#
# $bDone = true;
}
}
if (defined($pId))
{
if ($strRemote eq "out")
{
close($hPipeOut);
}
elsif ($strRemote eq "in" && defined($bCompress) && !$bCompress)
{
# confess "waiting for child";
close($hPipeIn);
}
waitpid($pId, 0);
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
if ($iExitStatus != 0)
{
confess &log(ERROR, "compression/decompression child process returned " . $iExitStatus);
}
}
# print "got out\n";
}
####################################################################################################################################
@@ -408,11 +554,11 @@ sub output_read
my $strError;
# print "error read wait\n";
#
#
# if (!eof($self->{hErr}))
# {
# $strError = $self->pipe_to_string($self->{hErr});
#
#
# if (defined($strError))
# {
# $bError = true;
@@ -446,19 +592,19 @@ sub output_read
confess &log(ERROR, (defined($strErrorPrefix) ? "${strErrorPrefix}" : "") .
(defined($strOutput) ? ": ${strOutput}" : ""), $iErrorCode);
}
$self->wait_pid();
if ($bOutputRequired && !defined($strOutput))
{
my $strError = $self->pipe_to_string($self->{hErr});
if (defined($strError))
{
$bError = true;
$strOutput = $strError;
}
# Capture any errors
if ($bError)
{

6
test/lib/BackRestTest/FileTest.pm
View File

@@ -805,14 +805,14 @@ sub BackRestFileTest
for (my $bDestinationCompressed = 0; $bDestinationCompressed <= 0; $bDestinationCompressed++)
{
my $strRemote = $bBackupRemote ? 'backup' : $bDbRemote ? 'db' : undef;
# $oRemote = BackRest::Remote->new
# (
# strHost => $strHost,
# strUser => $strUser,
# strCommand => $strCommand,
# );
my $oFile = BackRest::File->new
(
strStanza => "db",
@@ -862,7 +862,7 @@ sub BackRestFileTest
{
system("echo 'TESTDATA' > ${strSourceFile}");
}
my $strSourceHash = $oFile->hash(PATH_ABSOLUTE, $strSourceFile);
# Run file copy in an eval block because some errors are expected