db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

All copy() cases generate checksums during copy.

Browse Source 
Unit tests for bAppendChecksum option for copy().
This commit is contained in:
David Steele 2015-03-01 22:23:52 -05:00
parent 83658a4778
commit 23102f19e5
5 changed files with 297 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@ -27,21 +27,21 @@ Primary PgBackRest features:
* Added restore functionality.
* Added option (--no-start-stop) to allow backups when Postgres is shut down. If postmaster.pid is present then --force is required to make the backup run (though if Postgres is running an inconsistent backup will likely be created). This option was added primarily for the purpose of unit testing, but there may be applications in the real world as well.
* De/compression is now performed without threads and checksum/size is calculated in stream. That means file checksums are no longer optional.
* Removed dependency on Moose. It wasn't being used extensively and makes for longer startup times.
* Added option (--no-start-stop) to allow backups when Postgres is shut down. If postmaster.pid is present then --force is required to make the backup run (though if Postgres is running an inconsistent backup will likely be created). This option was added primarily for the purpose of unit testing, but there may be applications in the real world as well.
* Fixed broken checksums and now they work with normal and resumed backups. Finally realized that checksums and checksum deltas should be functionally separated and this simplied a number of things. Issue #28 has been created for checksum deltas.
* Fixed an issue where a backup could be resumed from an aborted backup that didn't have the same type and prior backup.
* More comprehensive backup unit tests.
* Removed dependency on Moose. It wasn't being used extensively and makes for longer startup times.
* Checksum for backup.manifest to detect corrupted/modified manifest.
* Link (called latest) always points to the last backup. This has been added for convenience and to make restore simpler.
* De/compression is now performed without threads and checksum/size are calculated in stream. That means file checksums are not longer optional.
* Checksum for backup.manifest to detect corrupted/modified manifest.
* More comprehensive backup unit tests.
### v0.30: core restructuring and unit tests

6
bin/pg_backrest_remote.pl
View File

@ -111,7 +111,8 @@ while ($strCommand ne OP_EXIT)
($bResult, $strChecksum, $iFileSize) =
$oFile->copy(PIPE_STDIN, undef,
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
undef, param_get(\%oParamHash, 'destination_compress'),
param_get(\%oParamHash, 'source_compressed'),
param_get(\%oParamHash, 'destination_compress'),
undef, undef,
param_get(\%oParamHash, 'permission', false),
param_get(\%oParamHash, 'destination_path_create'),
@ -125,7 +126,8 @@ while ($strCommand ne OP_EXIT)
($bResult, $strChecksum, $iFileSize) =
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
PIPE_STDOUT, undef,
param_get(\%oParamHash, 'source_compressed'), undef);
param_get(\%oParamHash, 'source_compressed'),
param_get(\%oParamHash, 'destination_compress'));
}
$oRemote->output_write(($bResult ? 'Y' : 'N') . " " . (defined($strChecksum) ? $strChecksum : '?') . " " .

75
lib/BackRest/File.pm
View File

@ -1427,6 +1427,7 @@ sub copy
{
$oParamHash{source_file} = $strSourceOp;
$oParamHash{source_compressed} = $bSourceCompressed;
$oParamHash{destination_compress} = $bDestinationCompress;
$hIn = $self->{oRemote}->{hOut};
}
@ -1445,6 +1446,7 @@ sub copy
else
{
$oParamHash{destination_file} = $strDestinationOp;
$oParamHash{source_compressed} = $bSourceCompressed;
$oParamHash{destination_compress} = $bDestinationCompress;
$oParamHash{destination_path_create} = $bDestinationPathCreate;
@ -1543,20 +1545,29 @@ sub copy
# Check the result of the remote call
if (substr($strOutput, 0, 1) eq 'Y')
{
# If checksum/size is not already defined then get it from the remote
if (!defined($strChecksum) || !defined($iFileSize))
# If the operation was purely remote, get checksum/size
if ($strOperation eq OP_FILE_COPY ||
$strOperation eq OP_FILE_COPY_IN && $bSourceCompressed && !$bDestinationCompress)
{
my @stryToken = split(/ /, $strOutput);
if ($bDestinationRemote && ($stryToken[1] eq '?' || $stryToken[2] eq '?'))
# Checksum shouldn't already be set
if (defined($strChecksum) || defined($iFileSize))
{
confess &log(ERROR, "checksum/size should have been returned from remote: ${strOutput}");
confess &log(ASSERT, "checksum and size are already defined, but shouldn't be");
}
# Parse output and check to make sure tokens are defined
my @stryToken = split(/ /, $strOutput);
if (!defined($stryToken[1]) || !defined($stryToken[2]) ||
$stryToken[1] eq '?' && $stryToken[2] eq '?')
{
confess &log(ERROR, "invalid return from copy" . (defined($strOutput) ? ": ${strOutput}" : ''));
}
# Read the checksum and size
if ($stryToken[1] ne '?')
{
$strChecksum = $stryToken[1];
$iFileSize = $stryToken[2];
}
if ($stryToken[2] ne '?')
@ -1565,6 +1576,7 @@ sub copy
}
}
}
# Remote called returned false
else
{
$bResult = false;
@ -1594,23 +1606,28 @@ sub copy
# Else this is a local operation
else
{
# If the source is compressed and the destination is not then decompress
if ($bSourceCompressed && !$bDestinationCompress)
{
($strChecksum, $iFileSize) =
$self->{oRemote}->binary_xfer($hSourceFile, $hDestinationFile, 'in', undef, false, false);
}
# If the source is not compressed and the destination is then compress
elsif (!$bSourceCompressed && $bDestinationCompress)
if (!$bSourceCompressed && $bDestinationCompress)
{
($strChecksum, $iFileSize) =
$self->{oRemote}->binary_xfer($hSourceFile, $hDestinationFile, 'out', false, undef, false);
}
# Else straight copy
# If the source is compressed and the destination is not then decompress
elsif ($bSourceCompressed && !$bDestinationCompress)
{
($strChecksum, $iFileSize) =
$self->{oRemote}->binary_xfer($hSourceFile, $hDestinationFile, 'in', undef, false, false);
}
# Else both side are compressed, so copy capturing checksum
elsif ($bSourceCompressed)
{
($strChecksum, $iFileSize) =
$self->{oRemote}->binary_xfer($hSourceFile, $hDestinationFile, 'out', true, true, false);
}
else
{
cp($hSourceFile, $hDestinationFile)
or die confess &log(ERROR, "${strDebug}: unable to copy: " . $!);
($strChecksum, $iFileSize) =
$self->{oRemote}->binary_xfer($hSourceFile, $hDestinationFile, 'in', undef, true, false);
}
}
@ -1626,8 +1643,16 @@ sub copy
close($hDestinationFile) or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
}
# Checksum and file size should be set if the destination is not remote
if ($bResult &&
!(!$bSourceRemote && $bDestinationRemote && $bSourceCompressed) &&
(!defined($strChecksum) || !defined($iFileSize)))
{
confess &log(ASSERT, "${strDebug}: checksum or file size not set");
}
# Where the destination is local, set permissions, modification time, and perform move to final location
if (!$bDestinationRemote)
if ($bResult && !$bDestinationRemote)
{
# Set the file permission if required
if (defined($strMode))
@ -1649,25 +1674,15 @@ sub copy
$self->owner(PATH_ABSOLUTE, $strDestinationTmpOp, $strUser, $strGroup);
}
# Get the checksum and size if they are not already set
if (!defined($strChecksum) || !defined($iFileSize))
{
($strChecksum, $iFileSize) = $self->hash_size(PATH_ABSOLUTE, $strDestinationTmpOp, $bDestinationCompress);
}
# Replace checksum in destination filename (if exists)
if ($bAppendChecksum)
{
if (!defined($strChecksum))
{
confess &log(ERROR, "${strDebug}: unable to append unset checksum");
}
# Replace destination filename
if ($bDestinationCompress)
{
$strDestinationOp =
substr($strDestinationOp, 0, length($strDestinationOp) - length($self->{strCompressExtension}) - 1) .
'-' . $strChecksum . '.' . $self->{strCompressExtension};
'-' . $strChecksum . '.' . $self->{strCompressExtension};
}
else
{

224
lib/BackRest/Remote.pm
View File

@ -14,6 +14,7 @@ use File::Basename;
use POSIX ':sys_wait_h';
use Scalar::Util 'blessed';
use Compress::Raw::Zlib;
use IO::String;
use lib dirname($0) . '/../lib';
use BackRest::Exception;
@ -359,20 +360,23 @@ sub block_read
my $bProtocol = shift;
my $iBlockSize;
my $strMessage;
if ($bProtocol)
{
# Read the block header and make sure it's valid
my $strBlockHeader = $self->read_line($hIn);
if ($strBlockHeader !~ /^block -{0,1}[0-9]+$/)
if ($strBlockHeader !~ /^block -{0,1}[0-9]+( .*){0,1}$/)
{
$self->wait_pid();
confess "unable to read block header ${strBlockHeader}";
}
# Get block size from the header
$iBlockSize = trim(substr($strBlockHeader, index($strBlockHeader, ' ') + 1));
my @stryToken = split(/ /, $strBlockHeader);
$iBlockSize = $stryToken[1];
$strMessage = $stryToken[2];
# If block size is 0 or an error code then undef the buffer
if ($iBlockSize <= 0)
@ -414,7 +418,7 @@ sub block_read
}
# Return the block size
return $iBlockSize;
return $iBlockSize, $strMessage;
}
####################################################################################################################################
@ -429,6 +433,7 @@ sub block_write
my $tBlockRef = shift;
my $iBlockSize = shift;
my $bProtocol = shift;
my $strMessage = shift;
# If block size is not defined, get it from buffer length
$iBlockSize = defined($iBlockSize) ? $iBlockSize : length($$tBlockRef);
@ -436,7 +441,7 @@ sub block_write
# Write block header to the protocol stream
if ($bProtocol)
{
$self->write_line($hOut, "block ${iBlockSize}");
$self->write_line($hOut, "block ${iBlockSize}" . (defined($strMessage) ? " ${strMessage}" : ''));
}
# Write block if size > 0
@ -535,9 +540,10 @@ sub binary_xfer
# Default protocol to true
$bProtocol = defined($bProtocol) ? $bProtocol : true;
my $strMessage = undef;
# Checksum and size
my $oSHA = undef;
my $strChecksum = undef;
my $iFileSize = undef;
# Read from the protocol stream
@ -551,9 +557,13 @@ sub binary_xfer
my $tUncompressedBuffer;
my $iUncompressedBufferSize;
# Initialize checksum and filesize
$oSHA = Digest::SHA->new('sha1');
$iFileSize = 0;
# Initialize SHA
my $oSHA;
if (!$bProtocol)
{
$oSHA = Digest::SHA->new('sha1');
}
# Initialize inflate object and check for errors
my ($oZLib, $iZLibStatus) =
@ -568,7 +578,14 @@ sub binary_xfer
do
{
# Read a block from the input stream
$iBlockSize = $self->block_read($hIn, \$tCompressedBuffer, $bProtocol);
($iBlockSize, $strMessage) = $self->block_read($hIn, \$tCompressedBuffer, $bProtocol);
# Process protocol messages
if (defined($strMessage) && $strMessage eq 'nochecksum')
{
$oSHA = Digest::SHA->new('sha1');
undef($strMessage);
}
# If the block contains data, decompress it
if ($iBlockSize > 0)
@ -583,14 +600,19 @@ sub binary_xfer
# If status is ok, write the data
if ($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR || $iZLibStatus == Z_STREAM_END)
{
# Add data to filesize and checksum
$iFileSize += $iUncompressedBufferSize;
$oSHA->add($tUncompressedBuffer);
# Write data if hOut is defined
if (defined($hOut))
if ($iUncompressedBufferSize > 0)
{
$self->stream_write($hOut, \$tUncompressedBuffer, $iUncompressedBufferSize);
# Add data to checksum
if (defined($oSHA))
{
$oSHA->add($tUncompressedBuffer);
}
# Write data if hOut is defined
if (defined($hOut))
{
$self->stream_write($hOut, \$tUncompressedBuffer, $iUncompressedBufferSize);
}
}
}
# Else error, exit so it can be handled
@ -610,6 +632,13 @@ sub binary_xfer
{
confess &log(ERROR, "unable to inflate stream: ${iZLibStatus}");
}
# Get checksum and total uncompressed bytes written
if (defined($oSHA))
{
$strChecksum = $oSHA->hexdigest();
$iFileSize = $oZLib->total_out();
};
}
# If the destination should be compressed then just write out the already compressed stream
else
@ -617,19 +646,41 @@ sub binary_xfer
my $iBlockSize;
my $tBuffer;
# Initialize checksum and size
my $oSHA;
if (!$bProtocol)
{
$oSHA = Digest::SHA->new('sha1');
$iFileSize = 0;
}
do
{
# Read a block from the protocol stream
$iBlockSize = $self->block_read($hIn, \$tBuffer, $bProtocol);
($iBlockSize, $strMessage) = $self->block_read($hIn, \$tBuffer, $bProtocol);
# If the block contains data, write it
if ($iBlockSize > 0)
{
# Add data to checksum and size
if (!$bProtocol)
{
$oSHA->add($tBuffer);
$iFileSize += $iBlockSize;
}
$self->stream_write($hOut, \$tBuffer, $iBlockSize);
undef($tBuffer);
}
}
while ($iBlockSize > 0);
# Get checksum
if (!$bProtocol)
{
$strChecksum = $oSHA->hexdigest();
};
}
}
# Read from file input stream
@ -643,8 +694,14 @@ sub binary_xfer
my $iCompressedBufferSize;
my $tUncompressedBuffer;
# Initialize message to indicate that a checksum will be sent
if ($bProtocol && defined($hOut))
{
$strMessage = 'checksum';
}
# Initialize checksum
$oSHA = Digest::SHA->new('sha1');
my $oSHA = Digest::SHA->new('sha1');
# Initialize inflate object and check for errors
my ($oZLib, $iZLibStatus) =
@ -676,8 +733,9 @@ sub binary_xfer
# The compressed data is larger than block size, then write
if ($iCompressedBufferSize > $self->{iBlockSize})
{
$self->block_write($hOut, \$tCompressedBuffer, $iCompressedBufferSize, $bProtocol);
$self->block_write($hOut, \$tCompressedBuffer, $iCompressedBufferSize, $bProtocol, $strMessage);
undef($tCompressedBuffer);
undef($strMessage);
}
}
# Else if error
@ -703,6 +761,10 @@ sub binary_xfer
confess &log(ERROR, "unable to deflate stream: ${iZLibStatus}");
}
# Get checksum and total uncompressed bytes written
$strChecksum = $oSHA->hexdigest();
$iFileSize = $oZLib->total_in();
# Write out the last block
if (defined($hOut))
{
@ -710,20 +772,51 @@ sub binary_xfer
if ($iCompressedBufferSize > 0)
{
$self->block_write($hOut, \$tCompressedBuffer, $iCompressedBufferSize, $bProtocol);
$self->block_write($hOut, \$tCompressedBuffer, $iCompressedBufferSize, $bProtocol, $strMessage);
undef($strMessage);
}
$self->block_write($hOut, undef, 0, $bProtocol);
$self->block_write($hOut, undef, 0, $bProtocol, "${strChecksum}-${iFileSize}");
}
# Get total uncompressed bytes written
$iFileSize = $oZLib->total_in();
}
# If source is already compressed or transfer is not compressed then just read the stream
else
{
my $iBlockSize;
my $tBuffer;
my $tCompressedBuffer;
my $tUncompressedBuffer;
my $iUncompressedBufferSize;
my $oSHA;
my $oZLib;
my $iZLibStatus;
# If the destination will be compressed setup deflate
if ($bDestinationCompress)
{
if ($bProtocol)
{
$strMessage = 'checksum';
}
# Initialize checksum and size
$oSHA = Digest::SHA->new('sha1');
$iFileSize = 0;
# Initialize inflate object and check for errors
($oZLib, $iZLibStatus) =
new Compress::Raw::Zlib::Inflate(WindowBits => WANT_GZIP, Bufsize => $self->{iBlockSize}, LimitOutput => 1);
if ($iZLibStatus != Z_OK)
{
confess &log(ERROR, "unable create a inflate object: ${iZLibStatus}");
}
}
# Initialize message to indicate that a checksum will not be sent
elsif ($bProtocol)
{
$strMessage = 'nochecksum';
}
# Read input
do
@ -733,18 +826,93 @@ sub binary_xfer
# Write a block if size > 0
if ($iBlockSize > 0)
{
$self->block_write($hOut, \$tBuffer, $iBlockSize, $bProtocol);
$self->block_write($hOut, \$tBuffer, $iBlockSize, $bProtocol, $strMessage);
undef($strMessage);
}
# Decompress the buffer to calculate checksum/size
if ($bDestinationCompress)
{
# If the block contains data, decompress it
if ($iBlockSize > 0)
{
# Copy file buffer to compressed buffer
if (defined($tCompressedBuffer))
{
$tCompressedBuffer .= $tBuffer;
}
else
{
$tCompressedBuffer = $tBuffer;
}
# Keep looping while there is more to decompress
do
{
# Decompress data
$iZLibStatus = $oZLib->inflate($tCompressedBuffer, $tUncompressedBuffer);
$iUncompressedBufferSize = length($tUncompressedBuffer);
# If status is ok, write the data
if ($iZLibStatus == Z_OK || $iZLibStatus == Z_BUF_ERROR || $iZLibStatus == Z_STREAM_END)
{
if ($iUncompressedBufferSize > 0)
{
$oSHA->add($tUncompressedBuffer);
$iFileSize += $iUncompressedBufferSize;
}
}
# Else error, exit so it can be handled
else
{
$iBlockSize = 0;
last;
}
}
while ($iZLibStatus == Z_OK && $iUncompressedBufferSize > 0);
}
}
}
while ($iBlockSize > 0);
# Write 0 block to indicate end of stream
$self->block_write($hOut, undef, 0, $bProtocol);
# Check decompression get checksum
if ($bDestinationCompress)
{
# Make sure the decompression succeeded (iBlockSize < 0 indicates remote error, handled later)
if ($iBlockSize == 0 && $iZLibStatus != Z_STREAM_END)
{
confess &log(ERROR, "unable to inflate stream: ${iZLibStatus}");
}
# Get checksum
$strChecksum = $oSHA->hexdigest();
# Set protocol message
if ($bProtocol)
{
$strMessage = "${strChecksum}-${iFileSize}";
}
}
# If protocol write
if ($bProtocol)
{
# Write 0 block to indicate end of stream
$self->block_write($hOut, undef, 0, $bProtocol, $strMessage);
}
}
}
# If message is defined the the checksum and size should be in it
if (defined($strMessage))
{
my @stryToken = split(/-/, $strMessage);
$strChecksum = $stryToken[0];
$iFileSize = $stryToken[1];
}
# Return the checksum and size if they are available
return (defined($oSHA) ? $oSHA->hexdigest() : undef), $iFileSize;
return $strChecksum, $iFileSize;
}
####################################################################################################################################

59
test/lib/BackRestTest/FileTest.pm
View File

@ -1062,8 +1062,11 @@ sub BackRestTestFile_Test
# Loop through source ignore/require
for (my $bSourceIgnoreMissing = 0; $bSourceIgnoreMissing <= !$bLarge; $bSourceIgnoreMissing++)
{
# Loop through checksum append
for (my $bChecksumAppend = 0; $bChecksumAppend <= !$bLarge; $bChecksumAppend++)
{
# Loop through source compression
for (my $bSourceCompressed = 0; $bSourceCompressed <= !$bSourceMissing; $bSourceCompressed++)
for (my $bSourceCompressed = 0; $bSourceCompressed <= !$bSourceMissing; $bSourceCompressed++)
{
# Loop through destination compression
for (my $bDestinationCompress = 0; $bDestinationCompress <= !$bSourceMissing; $bDestinationCompress++)
@ -1083,7 +1086,8 @@ sub BackRestTestFile_Test
"srcignmiss ${bSourceIgnoreMissing}, srccmp $bSourceCompressed, " .
'dstpth ' .
(defined($strRemote) && $strRemote eq $strDestinationPath ? 'rmt' : 'lcl') .
":${strDestinationPath}, dstcmp $bDestinationCompress")) {next}
":${strDestinationPath}, chkapp ${bChecksumAppend}, " .
"dstcmp $bDestinationCompress")) {next}
# Setup test directory
BackRestTestFile_Setup(false);
@ -1117,7 +1121,21 @@ sub BackRestTestFile_Test
system("echo 'TESTDATA' > ${strSourceFile}");
}
($strSourceHash, $iSourceSize) = $oFile->hash_size(PATH_ABSOLUTE, $strSourceFile);
if ($bLarge == 1)
{
$strSourceHash = 'c2e63b6a49d53a53d6df1aa6b70c7c16747ca099';
$iSourceSize = 16777216;
}
elsif ($bLarge == 2)
{
$strSourceHash = '1c7e00fd09b9dd11fc2966590b3e3274645dd031';
$iSourceSize = 16777216;
}
else
{
$strSourceHash = '06364afe79d801433188262478a76d19777ef351';
$iSourceSize = 9;
}
if ($bSourceCompressed)
{
@ -1140,7 +1158,8 @@ sub BackRestTestFile_Test
$oFile->copy($strSourcePathType, $strSourceFile,
$strDestinationPathType, $strDestinationFile,
$bSourceCompressed, $bDestinationCompress,
$bSourceIgnoreMissing, undef, '0700');
$bSourceIgnoreMissing, undef, '0700', false, undef, undef,
$bChecksumAppend);
};
# Check for errors after copy
@ -1183,6 +1202,24 @@ sub BackRestTestFile_Test
confess 'expected source file missing error';
}
if (!defined($strCopyHash))
{
confess 'copy hash must be defined';
}
if ($bChecksumAppend)
{
if ($bDestinationCompress)
{
$strDestinationFile =
substr($strDestinationFile, 0, length($strDestinationFile) -3) . "-${strSourceHash}.gz";
}
else
{
$strDestinationFile .= '-' . $strSourceHash;
}
}
unless (-e $strDestinationFile)
{
confess "could not find destination file ${strDestinationFile}";
@ -1200,19 +1237,14 @@ sub BackRestTestFile_Test
my ($strDestinationHash, $iDestinationSize) = $oFile->hash_size(PATH_ABSOLUTE, $strDestinationTest);
if ($strSourceHash ne $strDestinationHash)
if ($strSourceHash ne $strDestinationHash || $strSourceHash ne $strCopyHash)
{
confess "source ${strSourceHash} and destination ${strDestinationHash} file hashes do not match";
confess "source ${strSourceHash}, copy ${strCopyHash} and destination ${strDestinationHash} file hashes do not match";
}
# if ((!defined($strCopyHash) || !defined($iCopySize)) && !($bSourceCompressed && $bDestinationCompress))
# {
# confess "copy hash/size must be set unless source and destination are compressed";
# }
if (defined($strCopyHash) && $strSourceHash ne $strCopyHash)
if ($iSourceSize != $iDestinationSize || $iSourceSize != $iCopySize)
{
confess "source ${strSourceHash} and copy ${strCopyHash} file hashes do not match";
confess "source ${iSourceSize}, copy ${iCopySize} and destination ${iDestinationSize} sizes do not match";
}
}
}
@ -1221,6 +1253,7 @@ sub BackRestTestFile_Test
}
}
}
}
}
}
}