db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-09-16 09:06:18 +02:00
Code Issues Releases Activity

Working on calculating hash and file size in binary_xfer().

Browse Source
This commit is contained in:
David Steele
2015-02-28 15:54:49 -05:00
parent 0f94476a1f
commit 260a6cb8f1
5 changed files with 60 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
lib/BackRest/File.pm
View File

@@ -1334,6 +1334,10 @@ sub copy
my $strDestinationTmpOp = $strDestinationPathType eq PIPE_STDOUT ?
undef : $self->path_get($strDestinationPathType, $strDestinationFile, true);
# Checksum and size variables
my $strChecksum = undef;
my $iFileSize = undef;
# Set debug string and log
my $strDebug = ($bSourceRemote ? ' remote' : ' local') . " ${strSourcePathType}" .
(defined($strSourceFile) ? ":${strSourceOp}" : '') .
@@ -1367,7 +1371,7 @@ sub copy
if ($bIgnoreMissingSource && $strDestinationPathType ne PIPE_STDOUT)
{
return false;
return false, undef, undef;
}
}
@@ -1535,7 +1539,8 @@ sub copy
# Transfer the file (skip this for copies where both sides are remote)
if ($strOperation ne OP_FILE_COPY)
{
$self->{oRemote}->binary_xfer($hIn, $hOut, $strRemote, $bSourceCompressed, $bDestinationCompress);
($strChecksum, $iFileSize) =
$self->{oRemote}->binary_xfer($hIn, $hOut, $strRemote, $bSourceCompressed, $bDestinationCompress);
}
# If this is the controlling process then wait for OK from remote
@@ -1562,25 +1567,16 @@ sub copy
close($hDestinationFile) or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
unlink($strDestinationTmpOp) or confess &log(ERROR, "cannot remove file ${strDestinationTmpOp}");
return false;
return false, undef, undef;
}
# Otherwise report the error. I don't like this method of error reporting - raising the exception object makes the
# error hard to interpret and hides the error stack, raising test loses the error code.
if ($oMessage->isa('BackRest::Exception'))
{
confess $oMessage->message();
}
else
{
confess $oMessage;
}
confess $oMessage;
}
# If this was a remote copy, then return the result
if ($strOperation eq OP_FILE_COPY)
{
return false; #$strOutput eq 'N' ? true : false;
return false, undef, undef;
}
}
}
@@ -1644,7 +1640,7 @@ sub copy
$self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, true);
}
return true;
return true, $strChecksum, $iFileSize;
}
1;

21
lib/BackRest/Remote.pm
View File

@@ -38,7 +38,7 @@ use constant
####################################################################################################################################
use constant
{
DEFAULT_BLOCK_SIZE => 4194304
DEFAULT_BLOCK_SIZE => 8192
};
####################################################################################################################################
@@ -523,7 +523,10 @@ sub binary_xfer
my $strBlockHeader;
my $strBlock;
my $strBlockBuffer;
my $oGzip;
my $oGzip = undef;
my $oSHA = undef;
my $iFileSize = undef;
my $bFirst = true;
# Both the in and out streams must be defined
@@ -552,6 +555,9 @@ sub binary_xfer
# If this is the first block then initialize Gunzip
if ($bFirst)
{
$oSHA = Digest::SHA->new('sha1');
$iFileSize = 0;
if ($iBlockSize == 0)
{
&log(ASSERT, 'first protocol block is zero');
@@ -595,6 +601,9 @@ sub binary_xfer
# Write out the uncompressed bytes if there are any
if ($iUncompressedTotal > 0)
{
$oSHA->add($strBlock);
$iFileSize += $iUncompressedTotal;
$self->stream_write($hOut, \$strBlock, $iUncompressedTotal);
undef($strBlock);
}
@@ -634,6 +643,9 @@ sub binary_xfer
# Create the gzip object
if ($bFirst)
{
$oSHA = Digest::SHA->new('sha1');
$iFileSize = 0;
$oGzip = new IO::Compress::Gzip(\$strBlock, Append => 1)
or confess "IO::Compress::Gzip failed: $GzipError";
@@ -643,6 +655,8 @@ sub binary_xfer
# Read a block from the stream
$iBlockBufferIn = $self->stream_read($hIn, \$strBlockBuffer, $iBlockSize);
$oSHA->add($strBlockBuffer);
$iFileSize += $iBlockBufferIn;
# If block size > 0 then compress
if ($iBlockBufferIn > 0)
@@ -688,6 +702,9 @@ sub binary_xfer
}
}
}
# Return the checksum and size if they are available
return defined($oSHA) ? $oSHA->hexdigest() : undef, $iFileSize;
}
####################################################################################################################################

0
test/data/test.archive-complex.bin → test/data/test.archive1.bin
View File

0
test/data/test.archive.bin → test/data/test.archive2.bin
View File

53
test/lib/BackRestTest/FileTest.pm
View File

@@ -1021,6 +1021,9 @@ sub BackRestTestFile_Test
{
$iRun = 0;
# Loop through small/large
for (my $bLarge = false; $bLarge <= 2; $bLarge++)
{
# Loop through backup local vs remote
for (my $bBackupRemote = 0; $bBackupRemote <= 1; $bBackupRemote++)
{
@@ -1045,26 +1048,23 @@ sub BackRestTestFile_Test
defined($strRemote) ? $oRemote : undef
);
# Loop through source compression
for (my $bSourceCompressed = 0; $bSourceCompressed <= 1; $bSourceCompressed++)
{
# Loop through destination compression
for (my $bDestinationCompress = 0; $bDestinationCompress <= 1; $bDestinationCompress++)
{
# Loop through source path types
for (my $bSourcePathType = 0; $bSourcePathType <= 1; $bSourcePathType++)
{
# Loop through destination path types
for (my $bDestinationPathType = 0; $bDestinationPathType <= 1; $bDestinationPathType++)
{
# Loop through source ignore/require
for (my $bSourceIgnoreMissing = 0; $bSourceIgnoreMissing <= 1; $bSourceIgnoreMissing++)
{
# Loop through source missing/present
for (my $bSourceMissing = 0; $bSourceMissing <= 1; $bSourceMissing++)
for (my $bSourceMissing = 0; $bSourceMissing <= !$bLarge; $bSourceMissing++)
{
# Loop through small/large
for (my $bLarge = false; $bLarge <= defined($strRemote) && !$bSourceMissing; $bLarge++)
# Loop through source ignore/require
for (my $bSourceIgnoreMissing = 0; $bSourceIgnoreMissing <= !$bLarge; $bSourceIgnoreMissing++)
{
# Loop through source compression
for (my $bSourceCompressed = 0; $bSourceCompressed <= !$bSourceMissing; $bSourceCompressed++)
{
# Loop through destination compression
for (my $bDestinationCompress = 0; $bDestinationCompress <= !$bSourceMissing; $bDestinationCompress++)
{
my $strSourcePathType = $bSourcePathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
my $strSourcePath = $bSourcePathType ? 'db' : 'backup';
@@ -1073,13 +1073,12 @@ sub BackRestTestFile_Test
my $strDestinationPath = $bDestinationPathType ? 'db' : 'backup';
if (!BackRestTestCommon_Run(++$iRun,
'rmt ' .
"lrg ${bLarge}, rmt " .
(defined($strRemote) && ($strRemote eq $strSourcePath ||
$strRemote eq $strDestinationPath) ? 1 : 0) .
", lrg ${bLarge}, " .
'srcpth ' . (defined($strRemote) && $strRemote eq $strSourcePath ? 'rmt' : 'lcl') .
":${strSourcePath}, srccmp $bSourceCompressed, srcmiss ${bSourceMissing}, " .
"srcignmiss ${bSourceIgnoreMissing}, " .
', srcpth ' . (defined($strRemote) && $strRemote eq $strSourcePath ? 'rmt' : 'lcl') .
":${strSourcePath}, srcmiss ${bSourceMissing}, " .
"srcignmiss ${bSourceIgnoreMissing}, srccmp $bSourceCompressed, " .
'dstpth ' .
(defined($strRemote) && $strRemote eq $strDestinationPath ? 'rmt' : 'lcl') .
":${strDestinationPath}, dstcmp $bDestinationCompress")) {next}
@@ -1092,6 +1091,9 @@ sub BackRestTestFile_Test
my $strSourceFile = "${strTestPath}/${strSourcePath}/test-source";
my $strDestinationFile = "${strTestPath}/${strDestinationPath}/test-destination";
my $strCopyHash;
my $iCopySize;
# Create the compressed or uncompressed test file
my $strSourceHash;
@@ -1102,7 +1104,7 @@ sub BackRestTestFile_Test
$strSourceFile .= '.bin';
$strDestinationFile .= '.bin';
BackRestTestCommon_Execute('cp ' . BackRestTestCommon_DataPathGet() . "/test.archive.bin ${strSourceFile}");
BackRestTestCommon_Execute('cp ' . BackRestTestCommon_DataPathGet() . "/test.archive${bLarge}.bin ${strSourceFile}");
}
else
{
@@ -1131,11 +1133,11 @@ sub BackRestTestFile_Test
eval
{
$bReturn = $oFile->copy($strSourcePathType, $strSourceFile,
$strDestinationPathType, $strDestinationFile,
$bSourceCompressed, $bDestinationCompress,
$bSourceIgnoreMissing, undef,
'0700');
($bReturn, $strCopyHash, $iCopySize) =
$oFile->copy($strSourcePathType, $strSourceFile,
$strDestinationPathType, $strDestinationFile,
$bSourceCompressed, $bDestinationCompress,
$bSourceIgnoreMissing, undef, '0700');
};
# Check for errors after copy
@@ -1199,6 +1201,11 @@ sub BackRestTestFile_Test
{
confess "source ${strSourceHash} and destination ${strDestinationHash} file hashes do not match";
}
if (defined($strCopyHash) && $strSourceHash ne $strCopyHash)
{
confess "source ${strSourceHash} and copy ${strCopyHash} file hashes do not match";
}
}
}
}