db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-09-16 09:06:18 +02:00
Code Issues Releases Activity

ZLib stuff starting to look good. All references removed from File and using binary_xfer for all de/compression.

Browse Source
This commit is contained in:
David Steele
2015-03-01 13:41:35 -05:00
parent 7ede058b45
commit 77bc4238dc
2 changed files with 188 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
lib/BackRest/File.pm
View File

@@ -15,9 +15,6 @@ use File::Path qw(make_path remove_tree);
use Digest::SHA; use Digest::SHA;
use File::stat; use File::stat;
use Fcntl ':mode'; use Fcntl ':mode';
use IO::Compress::Gzip qw(gzip $GzipError);
use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
use IO::String;
use lib dirname($0) . '/../lib'; use lib dirname($0) . '/../lib';
use BackRest::Exception; use BackRest::Exception;
@@ -130,6 +127,11 @@ sub new
$self->{oRemote} = $oRemote; $self->{oRemote} = $oRemote;
$self->{iThreadIdx} = $iThreadIdx; $self->{iThreadIdx} = $iThreadIdx;
if (!defined($self->{strRemote}) || $self->{strRemote} eq NONE)
{
$self->{oRemote} = new BackRest::Remote();
}
# If remote is defined check parameters and open session # If remote is defined check parameters and open session
if (defined($self->{strRemote}) && $self->{strRemote} ne NONE) if (defined($self->{strRemote}) && $self->{strRemote} ne NONE)
{ {
@@ -523,25 +525,8 @@ sub compress
# Run locally # Run locally
else else
{ {
# Compress the file # Use copy to compress the file
if (!gzip($strPathOp => "${strPathOp}.gz")) $self->copy($strPathType, $strFile, $strPathType, "${strFile}.gz", false, true);
{
my $strError = "${strPathOp} could not be compressed:" . $!;
my $iErrorCode = COMMAND_ERR_FILE_READ;
if (!$self->exists($strPathType, $strFile))
{
$strError = "${strPathOp} does not exist";
$iErrorCode = COMMAND_ERR_FILE_MISSING;
}
if ($strPathType eq PATH_ABSOLUTE)
{
confess &log(ERROR, $strError, $iErrorCode);
}
confess &log(ERROR, "${strDebug}: " . $strError);
}
# Remove the old file # Remove the old file
unlink($strPathOp) unlink($strPathOp)
@@ -821,53 +806,8 @@ sub hash_size
if ($bCompressed) if ($bCompressed)
{ {
my $bFirst = true; ($strHash, $iSize) =
my $tCompressedBuffer; $self->{oRemote}->binary_xfer($hFile, undef, 'in', undef, false, false);
my $tUncompressedBuffer;
my $iBlockSize;
my $iBlockIn;
my $oGzip;
do
{
# Read a block from the file
$iBlockSize = sysread($hFile, $tCompressedBuffer, 4194304,
defined($tCompressedBuffer) ? length($tCompressedBuffer) : 0);
if (!defined($iBlockSize))
{
confess &log(ERROR, "${strFileOp} could not be read: " . $!);
}
# If this is the first block then initialize Gunzip
if ($bFirst)
{
# Initialize Gunzip
$oGzip = new IO::Uncompress::Gunzip(\$tCompressedBuffer, Transparent => 0, BlockSize => 4194304)
or confess "IO::Uncompress::Gunzip failed: $GunzipError";
# Clear first block flag
$bFirst = false;
}
# Loop while there is more data to uncompress
while (!$oGzip->eof())
{
# Decompress the block
$iBlockIn = $oGzip->read($tUncompressedBuffer);
if ($iBlockIn < 0)
{
confess &log(ERROR, "unable to decompress stream ($iBlockIn): ${GunzipError}");
}
$iSize += length($tUncompressedBuffer);
$oSHA->add($tUncompressedBuffer);
}
}
while ($iBlockSize > 0);
$oGzip->close();
} }
else else
{ {
@@ -888,11 +828,11 @@ sub hash_size
$oSHA->add($tBuffer); $oSHA->add($tBuffer);
} }
while ($iBlockSize > 0); while ($iBlockSize > 0);
$strHash = $oSHA->hexdigest();
} }
close($hFile); close($hFile);
$strHash = $oSHA->hexdigest();
} }
return $strHash, $iSize; return $strHash, $iSize;
@@ -1596,13 +1536,21 @@ sub copy
{ {
my @stryToken = split(/ /, $strOutput); my @stryToken = split(/ /, $strOutput);
if ($bDestinationRemote && ($stryToken[1] eq '?' || $stryToken[1] eq '?')) if ($bDestinationRemote && ($stryToken[1] eq '?' || $stryToken[2] eq '?'))
{ {
confess &log(ERROR, "checksum/size should have been returned from remote: ${strOutput}"); confess &log(ERROR, "checksum/size should have been returned from remote: ${strOutput}");
} }
$strChecksum = $stryToken[1]; if ($stryToken[1] ne '?')
$iFileSize = $stryToken[2]; {
$strChecksum = $stryToken[1];
$iFileSize = $stryToken[2];
}
if ($stryToken[2] ne '?')
{
$iFileSize = $stryToken[2];
}
} }
} }
else else
@@ -1637,14 +1585,16 @@ sub copy
# If the source is compressed and the destination is not then decompress # If the source is compressed and the destination is not then decompress
if ($bSourceCompressed && !$bDestinationCompress) if ($bSourceCompressed && !$bDestinationCompress)
{ {
gunzip($hSourceFile => $hDestinationFile) ($strChecksum, $iFileSize) =
or die confess &log(ERROR, "${strDebug}: unable to uncompress: " . $GunzipError); $self->{oRemote}->binary_xfer($hSourceFile, $hDestinationFile, 'in', undef, false, false);
} }
# If the source is not compressed and the destination is then compress
elsif (!$bSourceCompressed && $bDestinationCompress) elsif (!$bSourceCompressed && $bDestinationCompress)
{ {
gzip($hSourceFile => $hDestinationFile) ($strChecksum, $iFileSize) =
or die confess &log(ERROR, "${strDebug}: unable to compress: " . $GzipError); $self->{oRemote}->binary_xfer($hSourceFile, $hDestinationFile, 'out', false, undef, false);
} }
# Else straight copy
else else
{ {
cp($hSourceFile, $hDestinationFile) cp($hSourceFile, $hDestinationFile)
@@ -1690,8 +1640,11 @@ sub copy
# Move the file from tmp to final destination # Move the file from tmp to final destination
$self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, true); $self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, true);
# Get the checksum and size # Get the checksum and size if they are not already set
($strChecksum, $iFileSize) = $self->hash_size(PATH_ABSOLUTE, $strDestinationOp, $bDestinationCompress); if (!defined($strChecksum) || !defined($iFileSize))
{
($strChecksum, $iFileSize) = $self->hash_size(PATH_ABSOLUTE, $strDestinationOp, $bDestinationCompress);
}
} }
return $bResult, $strChecksum, $iFileSize; return $bResult, $strChecksum, $iFileSize;

249
lib/BackRest/Remote.pm
View File

@@ -15,6 +15,7 @@ use POSIX ':sys_wait_h';
use Scalar::Util 'blessed'; use Scalar::Util 'blessed';
use IO::Compress::Gzip qw($GzipError); use IO::Compress::Gzip qw($GzipError);
use IO::Uncompress::Gunzip qw($GunzipError); use IO::Uncompress::Gunzip qw($GunzipError);
use Compress::Raw::Zlib;
use lib dirname($0) . '/../lib'; use lib dirname($0) . '/../lib';
use BackRest::Exception; use BackRest::Exception;
@@ -357,50 +358,61 @@ sub block_read
my $self = shift; my $self = shift;
my $hIn = shift; my $hIn = shift;
my $strBlockRef = shift; my $strBlockRef = shift;
my $bProtocol = shift;
# Read the block header and make sure it's valid my $iBlockSize;
my $strBlockHeader = $self->read_line($hIn);
if ($strBlockHeader !~ /^block -{0,1}[0-9]+$/) if ($bProtocol)
{ {
$self->wait_pid(); # Read the block header and make sure it's valid
confess "unable to read block header ${strBlockHeader}"; my $strBlockHeader = $self->read_line($hIn);
}
# Get block size from the header if ($strBlockHeader !~ /^block -{0,1}[0-9]+$/)
my $iBlockSize = trim(substr($strBlockHeader, index($strBlockHeader, ' ') + 1)); {
$self->wait_pid();
confess "unable to read block header ${strBlockHeader}";
}
# If block size is 0 or an error code then undef the buffer # Get block size from the header
if ($iBlockSize <= 0) $iBlockSize = trim(substr($strBlockHeader, index($strBlockHeader, ' ') + 1));
{
undef($$strBlockRef); # If block size is 0 or an error code then undef the buffer
if ($iBlockSize <= 0)
{
undef($$strBlockRef);
}
# Else read the block
else
{
my $iBlockRead = 0;
my $iBlockIn = 0;
my $iOffset = defined($$strBlockRef) ? length($$strBlockRef) : 0;
# !!! Would be nice to modify this with a non-blocking read
# http://docstore.mik.ua/orelly/perl/cookbook/ch07_15.htm
# Read as many chunks as it takes to get the full block
while ($iBlockRead != $iBlockSize)
{
$iBlockIn = sysread($hIn, $$strBlockRef, $iBlockSize - $iBlockRead, $iBlockRead + $iOffset);
if (!defined($iBlockIn))
{
my $strError = $!;
$self->wait_pid();
confess "only read ${iBlockRead}/${iBlockSize} block bytes from remote" .
(defined($strError) ? ": ${strError}" : '');
}
$iBlockRead += $iBlockIn;
}
}
} }
# Else read the block
else else
{ {
my $iBlockRead = 0; $iBlockSize = $self->stream_read($hIn, $strBlockRef, $self->{iBlockSize},
my $iBlockIn = 0; defined($$strBlockRef) ? length($$strBlockRef) : 0);
my $iOffset = defined($$strBlockRef) ? length($$strBlockRef) : 0;
# !!! Would be nice to modify this with a non-blocking read
# http://docstore.mik.ua/orelly/perl/cookbook/ch07_15.htm
# Read as many chunks as it takes to get the full block
while ($iBlockRead != $iBlockSize)
{
$iBlockIn = sysread($hIn, $$strBlockRef, $iBlockSize - $iBlockRead, $iBlockRead + $iOffset);
if (!defined($iBlockIn))
{
my $strError = $!;
$self->wait_pid();
confess "only read ${iBlockRead}/${iBlockSize} block bytes from remote" .
(defined($strError) ? ": ${strError}" : '');
}
$iBlockRead += $iBlockIn;
}
} }
# Return the block size # Return the block size
@@ -418,13 +430,16 @@ sub block_write
my $hOut = shift; my $hOut = shift;
my $tBlockRef = shift; my $tBlockRef = shift;
my $iBlockSize = shift; my $iBlockSize = shift;
my $bProtocol = shift;
# If block size is not defined, get it from buffer length # If block size is not defined, get it from buffer length
$iBlockSize = defined($iBlockSize) ? $iBlockSize : length($$tBlockRef); $iBlockSize = defined($iBlockSize) ? $iBlockSize : length($$tBlockRef);
# Write block header to the protocol stream # Write block header to the protocol stream
$self->write_line($hOut, "block ${iBlockSize}"); if ($bProtocol)
# &log(INFO, "block ${iBlockSize}"); {
$self->write_line($hOut, "block ${iBlockSize}");
}
# Write block if size > 0 # Write block if size > 0
if ($iBlockSize > 0) if ($iBlockSize > 0)
@@ -500,6 +515,13 @@ sub binary_xfer
my $strRemote = shift; my $strRemote = shift;
my $bSourceCompressed = shift; my $bSourceCompressed = shift;
my $bDestinationCompress = shift; my $bDestinationCompress = shift;
my $bProtocol = shift;
# The input stream must be defined (output is optional)
if (!defined($hIn))
{
confess &log(ASSERT, 'hIn is not defined');
}
# If no remote is defined then set to none # If no remote is defined then set to none
if (!defined($strRemote)) if (!defined($strRemote))
@@ -513,27 +535,23 @@ sub binary_xfer
$bDestinationCompress = defined($bDestinationCompress) ? $bDestinationCompress : false; $bDestinationCompress = defined($bDestinationCompress) ? $bDestinationCompress : false;
} }
# Default protocol to true
$bProtocol = defined($bProtocol) ? $bProtocol : true;
# Working variables # Working variables
my $iBlockSize = $self->{iBlockSize}; my $iBlockSize = $self->{iBlockSize};
my $iBlockIn; my $iBlockIn;
my $iBlockInTotal = $iBlockSize;
my $iBlockBufferIn; my $iBlockBufferIn;
my $iBlockOut;
my $iBlockTotal = 0;
my $strBlockHeader;
my $strBlock; my $strBlock;
my $strBlockBuffer; my $strBlockBuffer;
my $oGzip = undef;
my $oSHA = undef;
my $iFileSize = undef; my $iFileSize = undef;
my $bFirst = true; my $oGzip = undef;
my $oZLib = undef;
my $iZLibStatus;
my $oSHA = undef;
# Both the in and out streams must be defined my $bFirst = true;
if (!defined($hIn) || !defined($hOut))
{
confess &log(ASSERT, 'hIn or hOut is not defined');
}
while (1) while (1)
{ {
@@ -544,9 +562,9 @@ sub binary_xfer
if (!$bDestinationCompress) if (!$bDestinationCompress)
{ {
# Read a block from the protocol stream # Read a block from the protocol stream
$iBlockSize = $self->block_read($hIn, \$strBlockBuffer); $iBlockSize = $self->block_read($hIn, \$strBlockBuffer, $bProtocol);
# If block size = -1 it means an error happened on the remote we need to exit so it can be returned. # If block size = -1 it means an error happened on the remote and we need to exit so it can be returned.
if ($iBlockSize == -1) if ($iBlockSize == -1)
{ {
last; last;
@@ -558,22 +576,30 @@ sub binary_xfer
$oSHA = Digest::SHA->new('sha1'); $oSHA = Digest::SHA->new('sha1');
$iFileSize = 0; $iFileSize = 0;
if ($iBlockSize == 0) # if ($iBlockSize == 0)
{ # {
&log(ASSERT, 'first protocol block is zero'); # &log(ASSERT, 'first protocol block is zero');
} # }
#
# Gunzip doesn't like to be initialized with just the header, so if the first block is 10 bytes then fetch # # Gunzip doesn't like to be initialized with just the header, so if the first block is 10 bytes then fetch
# another another block to make sure so is at least some payload. # # another another block to make sure so is at least some payload.
if ($iBlockSize <= 10) # if ($iBlockSize <= 10)
{ # {
$iBlockSize = $self->block_read($hIn, \$strBlockBuffer); # $iBlockSize = $self->block_read($hIn, \$strBlockBuffer, $bProtocol);
} # }
# Initialize Gunzip # Initialize Gunzip
$oGzip = new IO::Uncompress::Gunzip(\$strBlockBuffer, Append => 1, Transparent => 0, ($oZLib, $iZLibStatus) = new Compress::Raw::Zlib::Inflate(LimitOutput => 1, -WindowBits => WANT_GZIP,
BlockSize => $self->{iBlockSize}) -Bufsize => $self->{iBlockSize});
or confess "IO::Uncompress::Gunzip failed (${iBlockSize}): $GunzipError";
if ($iZLibStatus != Z_OK)
{
confess &log(ERROR, "unable create a inflation stream: ${iZLibStatus}");
}
# $oGzip = new IO::Uncompress::Gunzip(\$strBlockBuffer, Append => 1, Transparent => 0,
# BlockSize => $self->{iBlockSize})
# or confess "IO::Uncompress::Gunzip failed (${iBlockSize}): $GunzipError";
# Clear first block flag # Clear first block flag
$bFirst = false; $bFirst = false;
@@ -582,36 +608,67 @@ sub binary_xfer
# If the block contains data, decompress it # If the block contains data, decompress it
if ($iBlockSize > 0) if ($iBlockSize > 0)
{ {
my $iUncompressedTotal = 0; # my $iUncompressedTotal = 0;
# Loop while there is more data to uncompress # # Loop while there is more data to uncompress
while (!$oGzip->eof()) # while (!$oGzip->eof())
# {
# # Decompress the block
# $iBlockIn = $oGzip->read($strBlock);
#
# if ($iBlockIn < 0)
# {
# confess &log(ERROR, "unable to decompress stream ($iBlockIn): ${GunzipError}");
# }
#
# $iUncompressedTotal += $iBlockIn;
# }
do
{ {
# Decompress the block $iZLibStatus = $oZLib->inflate($strBlockBuffer, $strBlock);
$iBlockIn = $oGzip->read($strBlock);
if ($iBlockIn < 0) if ($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR || $iZLibStatus == Z_STREAM_END)
{ {
confess &log(ERROR, "unable to decompress stream ($iBlockIn): ${GunzipError}"); $iFileSize += length($strBlock);
$oSHA->add($strBlock);
if (defined($hOut))
{
$self->stream_write($hOut, \$strBlock);
}
}
else
{
$iBlockSize = 0;
last;
} }
$iUncompressedTotal += $iBlockIn;
} }
while ($iZLibStatus == Z_OK && length($strBlock));
# Write out the uncompressed bytes if there are any # # Write out the uncompressed bytes if there are any
if ($iUncompressedTotal > 0) # if ($iUncompressedTotal > 0)
{ # {
$oSHA->add($strBlock); # $oSHA->add($strBlock);
$iFileSize += $iUncompressedTotal; # $iFileSize += $iUncompressedTotal;
#
$self->stream_write($hOut, \$strBlock, $iUncompressedTotal); # if (defined($hOut))
undef($strBlock); # {
} # $self->stream_write($hOut, \$strBlock, $iUncompressedTotal);
# }
#
# undef($strBlock);
# }
} }
# Else close gzip
else # Make sure the decompression succeeded
if ($iBlockSize == 0)
{ {
$iBlockIn = $oGzip->close(); if ($iZLibStatus != Z_STREAM_END)
{
confess &log(ERROR, "unable to inflate stream: gzip returned ${iZLibStatus}");
}
last; last;
} }
} }
@@ -619,7 +676,7 @@ sub binary_xfer
else else
{ {
# Read a block from the protocol stream # Read a block from the protocol stream
$iBlockSize = $self->block_read($hIn, \$strBlock); $iBlockSize = $self->block_read($hIn, \$strBlock, $bProtocol);
# If the block contains data, write it # If the block contains data, write it
if ($iBlockSize > 0) if ($iBlockSize > 0)
@@ -669,9 +726,9 @@ sub binary_xfer
confess &log(ERROR, "IO::Compress::Gzip failed: $GzipError"); confess &log(ERROR, "IO::Compress::Gzip failed: $GzipError");
} }
if (defined($strBlock) && length($strBlock) > $self->{iBlockSize}) if (defined($hOut) && defined($strBlock) && length($strBlock) > $self->{iBlockSize})
{ {
$self->block_write($hOut, \$strBlock); $self->block_write($hOut, \$strBlock, undef, $bProtocol);
undef($strBlock); undef($strBlock);
} }
} }
@@ -680,8 +737,12 @@ sub binary_xfer
{ {
$oGzip->close(); $oGzip->close();
$self->block_write($hOut, \$strBlock); if (defined($hOut))
$self->block_write($hOut, undef, 0); {
$self->block_write($hOut, \$strBlock, undef, $bProtocol);
$self->block_write($hOut, undef, 0, $bProtocol);
}
last; last;
} }
} }
@@ -692,11 +753,11 @@ sub binary_xfer
if ($iBlockIn > 0) if ($iBlockIn > 0)
{ {
$self->block_write($hOut, \$strBlock, $iBlockIn); $self->block_write($hOut, \$strBlock, $iBlockIn, $bProtocol);
} }
else else
{ {
$self->block_write($hOut, undef, 0); $self->block_write($hOut, undef, 0, $bProtocol);
last; last;
} }
} }