db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-10-30 23:37:45 +02:00
Code Issues Releases Activity

File->copy now returns hash and size in all cases, though the local copies are not optimal. They just call hash_size().

Browse Source
This commit is contained in:
David Steele
2015-02-28 19:07:29 -05:00
parent 260a6cb8f1
commit 28326d6b4c
5 changed files with 150 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
bin/pg_backrest_remote.pl
View File

@@ -80,28 +80,54 @@ while ($strCommand ne OP_EXIT)
eval
{
# Copy a file to STDOUT
if ($strCommand eq OP_FILE_COPY_OUT)
# Copy file
if ($strCommand eq OP_FILE_COPY ||
$strCommand eq OP_FILE_COPY_IN ||
$strCommand eq OP_FILE_COPY_OUT)
{
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
PIPE_STDOUT, undef,
param_get(\%oParamHash, 'source_compressed'), undef);
my $bResult;
my $strChecksum;
my $iFileSize;
$oRemote->output_write();
}
# Copy a file from STDIN
elsif ($strCommand eq OP_FILE_COPY_IN)
{
$oFile->copy(PIPE_STDIN, undef,
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
undef, param_get(\%oParamHash, 'destination_compress'),
undef, undef,
param_get(\%oParamHash, 'permission', false),
param_get(\%oParamHash, 'destination_path_create'),
param_get(\%oParamHash, 'user', false),
param_get(\%oParamHash, 'group', false));
# Copy a file locally
if ($strCommand eq OP_FILE_COPY)
{
($bResult, $strChecksum, $iFileSize) =
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
param_get(\%oParamHash, 'source_compressed'),
param_get(\%oParamHash, 'destination_compress'),
param_get(\%oParamHash, 'ignore_missing_source', false),
undef,
param_get(\%oParamHash, 'permission', false),
param_get(\%oParamHash, 'destination_path_create') ? 'Y' : 'N',
param_get(\%oParamHash, 'user', false),
param_get(\%oParamHash, 'group', false));
}
# Copy a file from STDIN
elsif ($strCommand eq OP_FILE_COPY_IN)
{
($bResult, $strChecksum, $iFileSize) =
$oFile->copy(PIPE_STDIN, undef,
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
undef, param_get(\%oParamHash, 'destination_compress'),
undef, undef,
param_get(\%oParamHash, 'permission', false),
param_get(\%oParamHash, 'destination_path_create'),
param_get(\%oParamHash, 'user', false),
param_get(\%oParamHash, 'group', false));
}
# Copy a file to STDOUT
elsif ($strCommand eq OP_FILE_COPY_OUT)
{
($bResult, $strChecksum, $iFileSize) =
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
PIPE_STDOUT, undef,
param_get(\%oParamHash, 'source_compressed'), undef);
}
$oRemote->output_write();
$oRemote->output_write(($bResult ? 'Y' : 'N') . " " . (defined($strChecksum) ? $strChecksum : '?') . " " .
(defined($iFileSize) ? $iFileSize : '?'));
}
# List files in a path
elsif ($strCommand eq OP_FILE_LIST)
@@ -134,21 +160,6 @@ while ($strCommand ne OP_EXIT)
{
$oRemote->output_write($oFile->exists(PATH_ABSOLUTE, param_get(\%oParamHash, 'path')) ? 'Y' : 'N');
}
# Copy a file locally
elsif ($strCommand eq OP_FILE_COPY)
{
$oRemote->output_write(
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
param_get(\%oParamHash, 'source_compressed'),
param_get(\%oParamHash, 'destination_compress'),
param_get(\%oParamHash, 'ignore_missing_source', false),
undef,
param_get(\%oParamHash, 'permission', false),
param_get(\%oParamHash, 'destination_path_create')) ? 'Y' : 'N',
param_get(\%oParamHash, 'user', false),
param_get(\%oParamHash, 'group', false));
}
# Wait
elsif ($strCommand eq OP_FILE_WAIT)
{

52
lib/BackRest/Backup.pm
View File

@@ -1136,7 +1136,11 @@ sub backup_file_thread
my $lSize = 0; # Size of files currently copied by this thread
my $strLog; # Store the log message
my $strLogProgress; # Part of the log message that shows progress
my $oFileThread; # Thread local file object
my $bCopyResult; # Copy result
my $strCopyChecksum; # Copy checksum
my $lCopySize; # Copy Size
# If multi-threaded, then clone the file object
if ($bMulti)
@@ -1160,20 +1164,21 @@ sub backup_file_thread
if (!$oFileCopyMap{$strFile}{checksum_only})
{
# Output information about the file to be copied
$strLog = "thread ${iThreadIdx} backing up file $oFileCopyMap{$strFile}{db_file} (" .
file_size_format($oFileCopyMap{$strFile}{size}) .
($lSizeTotal > 0 ? ', ' . int($lSize * 100 / $lSizeTotal) . '%' : '') . ')';
$strLog = "thread ${iThreadIdx} backing up file";
# Copy the file from the database to the backup (will return false if the source file is missing)
unless($oFileThread->copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file} .
($bCompress ? '.' . $oFile->{strCompressExtension} : ''),
false, # Source is not compressed since it is the db directory
$bCompress, # Destination should be compressed based on backup settings
true, # Ignore missing files
$oFileCopyMap{$strFile}{modification_time}, # Set modification time
undef, # Do not set original permissions
true)) # Create the destiation directory if it does not exist
($bCopyResult, $strCopyChecksum, $lCopySize) =
$oFileThread->copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file} .
($bCompress ? '.' . $oFile->{strCompressExtension} : ''),
false, # Source is not compressed since it is the db directory
$bCompress, # Destination should be compressed based on backup settings
true, # Ignore missing files
$oFileCopyMap{$strFile}{modification_time}, # Set modification time
undef, # Do not set original permissions
true); # Create the destination directory if it does not exist
if (!$bCopyResult)
{
# If file is missing assume the database removed it (else corruption and nothing we can do!)
&log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file});
@@ -1196,40 +1201,37 @@ sub backup_file_thread
}
}
# Generate checksum for file if configured
if ($bChecksum && $lSize != 0)
{
# Generate the checksum
my $strChecksum = $oFileThread->hash(PATH_BACKUP_TMP,
$oFileCopyMap{$strFile}{backup_file} . ($bCompress ? '.gz' : ''), $bCompress);
$strLogProgress = "$oFileCopyMap{$strFile}{db_file} (" . file_size_format($lCopySize) .
($lSizeTotal > 0 ? ', ' . int($lSize * 100 / $lSizeTotal) . '%' : '') . ')';
# Generate checksum for file if configured
if ($bChecksum && $lCopySize != 0)
{
# Store checksum in the manifest
if ($bMulti)
{
# Write the checksum message into the master queue
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|" .
"$oFileCopyMap{$strFile}{file}|${strChecksum}");
"$oFileCopyMap{$strFile}{file}|${strCopyChecksum}");
}
else
{
# Write it directly
$oBackupManifest->set($oFileCopyMap{$strFile}{file_section}, $oFileCopyMap{$strFile}{file},
MANIFEST_SUBKEY_CHECKSUM, $strChecksum);
MANIFEST_SUBKEY_CHECKSUM, $strCopyChecksum);
}
# Output information about the file to be checksummed
if (!defined($strLog))
{
$strLog = "thread ${iThreadIdx} checksum-only $oFileCopyMap{$strFile}{db_file} (" .
file_size_format($oFileCopyMap{$strFile}{size}) .
($lSizeTotal > 0 ? ', ' . int($lSize * 100 / $lSizeTotal) . '%' : '') . ')';
$strLog = "thread ${iThreadIdx} checksum-only ${strLogProgress}";
}
&log(INFO, $strLog . " checksum ${strChecksum}");
&log(INFO, $strLog . " checksum ${strCopyChecksum}");
}
else
{
&log(INFO, $strLog);
&log(INFO, $strLog . ' ' . $strLogProgress);
}
&log(TRACE, "thread waiting for new file from queue");

74
lib/BackRest/File.pm
View File

@@ -758,6 +758,22 @@ sub hash
my $bCompressed = shift;
my $strHashType = shift;
my ($strHash, $iSize) = $self->hash_size($strPathType, $strFile, $bCompressed, $strHashType);
return $strHash;
}
####################################################################################################################################
# HASH_SIZE
####################################################################################################################################
sub hash_size
{
my $self = shift;
my $strPathType = shift;
my $strFile = shift;
my $bCompressed = shift;
my $strHashType = shift;
# Set defaults
$bCompressed = defined($bCompressed) ? $bCompressed : false;
$strHashType = defined($strHashType) ? $strHashType : 'sha1';
@@ -765,6 +781,7 @@ sub hash
# Set operation variables
my $strFileOp = $self->path_get($strPathType, $strFile);
my $strHash;
my $iSize = 0;
# Set operation and debug strings
my $strOperation = OP_FILE_HASH;
@@ -844,6 +861,7 @@ sub hash
confess &log(ERROR, "unable to decompress stream ($iBlockIn): ${GunzipError}");
}
$iSize += length($tUncompressedBuffer);
$oSHA->add($tUncompressedBuffer);
}
}
@@ -853,7 +871,23 @@ sub hash
}
else
{
$oSHA->addfile($hFile);
my $iBlockSize;
my $tBuffer;
do
{
# Read a block from the file
$iBlockSize = sysread($hFile, $tBuffer, 4194304);
if (!defined($iBlockSize))
{
confess &log(ERROR, "${strFileOp} could not be read: " . $!);
}
$iSize += $iBlockSize;
$oSHA->add($tBuffer);
}
while ($iBlockSize > 0);
}
close($hFile);
@@ -861,7 +895,7 @@ sub hash
$strHash = $oSHA->hexdigest();
}
return $strHash;
return $strHash, $iSize;
}
####################################################################################################################################
@@ -1337,6 +1371,7 @@ sub copy
# Checksum and size variables
my $strChecksum = undef;
my $iFileSize = undef;
my $bResult = true;
# Set debug string and log
my $strDebug = ($bSourceRemote ? ' remote' : ' local') . " ${strSourcePathType}" .
@@ -1551,7 +1586,29 @@ sub copy
eval
{
$strOutput = $self->{oRemote}->output_read($strOperation eq OP_FILE_COPY, $strDebug, true);
$strOutput = $self->{oRemote}->output_read(true, $strDebug, true);
# Check the result of the remote call
if (substr($strOutput, 0, 1) eq 'Y')
{
# If checksum/size is not already defined then get it from the remote
if (!defined($strChecksum) || !defined($iFileSize))
{
my @stryToken = split(/ /, $strOutput);
if ($bDestinationRemote && ($stryToken[1] eq '?' || $stryToken[1] eq '?'))
{
confess &log(ERROR, "checksum/size should have been returned from remote: ${strOutput}");
}
$strChecksum = $stryToken[1];
$iFileSize = $stryToken[2];
}
}
else
{
$bResult = false;
}
};
# If there is an error then evaluate
@@ -1572,12 +1629,6 @@ sub copy
confess $oMessage;
}
# If this was a remote copy, then return the result
if ($strOperation eq OP_FILE_COPY)
{
return false, undef, undef;
}
}
}
# Else this is a local operation
@@ -1638,9 +1689,12 @@ sub copy
# Move the file from tmp to final destination
$self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, true);
# Get the checksum and size
($strChecksum, $iFileSize) = $self->hash_size(PATH_ABSOLUTE, $strDestinationOp, $bDestinationCompress);
}
return true, $strChecksum, $iFileSize;
return $bResult, $strChecksum, $iFileSize;
}
1;

2
lib/BackRest/Remote.pm
View File

@@ -704,7 +704,7 @@ sub binary_xfer
}
# Return the checksum and size if they are available
return defined($oSHA) ? $oSHA->hexdigest() : undef, $iFileSize;
return (defined($oSHA) ? $oSHA->hexdigest() : undef), $iFileSize;
}
####################################################################################################################################

18
test/lib/BackRestTest/FileTest.pm
View File

@@ -339,6 +339,7 @@ sub BackRestTestFile_Test
my $strFile = "${strTestPath}/test.txt";
my $strSourceHash;
my $iSourceSize;
if ($bError)
{
@@ -347,7 +348,7 @@ sub BackRestTestFile_Test
elsif ($bExists)
{
system("echo 'TESTDATA' > ${strFile}");
$strSourceHash = $oFile->hash(PATH_BACKUP_ABSOLUTE, $strFile);
($strSourceHash, $iSourceSize) = $oFile->hash_size(PATH_BACKUP_ABSOLUTE, $strFile);
}
# Execute in eval in case of error
@@ -385,7 +386,7 @@ sub BackRestTestFile_Test
system("gzip -d ${strDestinationFile}") == 0 or die "could not decompress ${strDestinationFile}";
my $strDestinationHash = $oFile->hash(PATH_BACKUP_ABSOLUTE, $strFile);
my ($strDestinationHash, $iDestinationSize) = $oFile->hash_size(PATH_BACKUP_ABSOLUTE, $strFile);
if ($strSourceHash ne $strDestinationHash)
{
@@ -895,11 +896,12 @@ sub BackRestTestFile_Test
# Execute in eval in case of error
my $strHash;
my $iSize;
my $bErrorExpected = !$bExists || $bError || $bRemote;
eval
{
$strHash = $oFile->hash(PATH_BACKUP_ABSOLUTE, $strFile, $bCompressed)
($strHash, $iSize) = $oFile->hash_size(PATH_BACKUP_ABSOLUTE, $strFile, $bCompressed)
};
if ($@)
@@ -1096,6 +1098,7 @@ sub BackRestTestFile_Test
# Create the compressed or uncompressed test file
my $strSourceHash;
my $iSourceSize;
if (!$bSourceMissing)
{
@@ -1114,7 +1117,7 @@ sub BackRestTestFile_Test
system("echo 'TESTDATA' > ${strSourceFile}");
}
$strSourceHash = $oFile->hash(PATH_ABSOLUTE, $strSourceFile);
($strSourceHash, $iSourceSize) = $oFile->hash_size(PATH_ABSOLUTE, $strSourceFile);
if ($bSourceCompressed)
{
@@ -1195,13 +1198,18 @@ sub BackRestTestFile_Test
or die "could not decompress ${strDestinationFile}";
}
my $strDestinationHash = $oFile->hash(PATH_ABSOLUTE, $strDestinationTest);
my ($strDestinationHash, $iDestinationSize) = $oFile->hash_size(PATH_ABSOLUTE, $strDestinationTest);
if ($strSourceHash ne $strDestinationHash)
{
confess "source ${strSourceHash} and destination ${strDestinationHash} file hashes do not match";
}
# if ((!defined($strCopyHash) || !defined($iCopySize)) && !($bSourceCompressed && $bDestinationCompress))
# {
# confess "copy hash/size must be set unless source and destination are compressed";
# }
if (defined($strCopyHash) && $strSourceHash ne $strCopyHash)
{
confess "source ${strSourceHash} and copy ${strCopyHash} file hashes do not match";