db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-10-30 23:37:45 +02:00
Code Issues Releases Activity

Ignore missing source in copy() working.

Browse Source
This commit is contained in:
David Steele
2014-06-21 12:22:25 -04:00
parent e1903d53d0
commit 80206a28e9
3 changed files with 226 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bin/pg_backrest_remote.pl
View File

@@ -113,12 +113,12 @@ while ($strCommand ne OP_EXIT)
}
elsif ($strCommand eq OP_FILE_COPY)
{
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
param_get(\%oParamHash, 'source_compressed'),
param_get(\%oParamHash, 'destination_compress'));
$oRemote->output_write();
$oRemote->output_write(
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
param_get(\%oParamHash, 'source_compressed'),
param_get(\%oParamHash, 'destination_compress'),
param_get(\%oParamHash, 'ignore_missing_source', false)) ? 'Y' : 'N');
}
elsif ($strCommand eq OP_FILE_COPY_IN)
{

176
lib/BackRest/File.pm
View File

@@ -569,111 +569,6 @@ sub move
}
}
####################################################################################################################################
# PIPE_TO_STRING Function
#
# Copies data from a file handle into a string.
####################################################################################################################################
# sub pipe_to_string
# {
# my $self = shift;
# my $hOut = shift;
#
# my $strBuffer;
# my $hString = IO::String->new($strBuffer);
# $self->pipe($hOut, $hString);
#
# return $strBuffer;
# }
####################################################################################################################################
# PIPE Function
#
# Copies data from one file handle to another, optionally compressing or decompressing the data in stream.
####################################################################################################################################
# sub pipe
# {
# my $self = shift;
# my $hIn = shift;
# my $hOut = shift;
# my $bCompress = shift;
# my $bUncompress = shift;
#
# # If compression is requested and the file is not already compressed
# if (defined($bCompress) && $bCompress)
# {
# if (!gzip($hIn => $hOut))
# {
# confess $GzipError;
# }
# }
# # If no compression is requested and the file is already compressed
# elsif (defined($bUncompress) && $bUncompress)
# {
# if (!gunzip($hIn => $hOut))
# {
# confess $GunzipError;
# }
# }
# # Else it's a straight copy
# else
# {
# my $strBuffer;
# my $iResultRead;
# my $iResultWrite;
#
# # Read from the input handle
# while (($iResultRead = sysread($hIn, $strBuffer, BLOCK_SIZE)) != 0)
# {
# if (!defined($iResultRead))
# {
# confess $!;
# last;
# }
# else
# {
# # Write to the output handle
# $iResultWrite = syswrite($hOut, $strBuffer, $iResultRead);
#
# if (!defined($iResultWrite) || $iResultWrite != $iResultRead)
# {
# confess $!;
# last;
# }
# }
# }
# }
# }
####################################################################################################################################
# WAIT_PID
####################################################################################################################################
# sub wait_pid
# {
# my $self = shift;
# my $pId = shift;
# my $strCommand = shift;
# my $hIn = shift;
# my $hOut = shift;
# my $hErr = shift;
#
# # Close hIn
# close($hIn);
#
# # Read STDERR into a string
# my $strError = defined($hErr) ? $self->pipe_to_string($hErr) : "[unknown]";
#
# # Wait for the process to finish and report any errors
# waitpid($pId, 0);
# my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
#
# if ($iExitStatus != 0)
# {
# confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus . ": " .
# (defined($strError) ? $strError : "[unknown]"));
# }
# }
####################################################################################################################################
# EXISTS - Checks for the existence of a file, but does not imply that the file is readable/writeable.
#
@@ -780,14 +675,15 @@ sub copy
my $strDestinationTmpOp = $strDestinationPathType eq PIPE_STDOUT ?
undef : $self->path_get($strDestinationPathType, $strDestinationFile, true);
# Set operation and debug string
# my $strOperation = "File->copy[unknown]";
# Set debug string and log
my $strDebug = ($bSourceRemote ? " remote" : " local") . " ${strSourcePathType}" .
(defined($strSourceFile) ? ":${strSourceFile}" : "") .
" to" . ($bDestinationRemote ? " remote" : " local") . " ${strDestinationPathType}" .
(defined($strDestinationFile) ? ":${strDestinationFile}" : "") .
", source_compressed = " . ($bSourceCompressed ? "true" : "false") .
", destination_compress = " . ($bDestinationCompress ? "true" : "false");
", destination_compress = " . ($bDestinationCompress ? "true" : "false") .
", ignore_missing_source = " . ($bIgnoreMissingSource ? "true" : "false");
&log(DEBUG, OP_FILE_COPY . ": ${strDebug}");
# Open the source and destination files (if needed)
my $hSourceFile;
@@ -804,13 +700,22 @@ sub copy
{
# $strError = 'file is missing';
$iErrorCode = COMMAND_ERR_FILE_MISSING;
if ($bIgnoreMissingSource && $strDestinationPathType ne PIPE_STDOUT)
{
return false;
}
}
$strError = "cannot open source file ${strSourceOp}: " . $strError;
if ($strSourcePathType eq PATH_ABSOLUTE)
{
$self->{oRemote}->write_line(*STDOUT, "block 0");
if ($strDestinationPathType eq PIPE_STDOUT)
{
$self->{oRemote}->write_line(*STDOUT, "block 0");
}
confess &log(ERROR, $strError, $iErrorCode);
}
@@ -882,12 +787,21 @@ sub copy
$oParamHash{source_compressed} = $bSourceCompressed;
$oParamHash{destination_file} = $strDestinationOp;
$oParamHash{destination_compress} = $bDestinationCompress;
if ($bIgnoreMissingSource)
{
$oParamHash{ignore_missing_source} = $bIgnoreMissingSource;
}
}
# Build debug string
$strDebug = "${strOperation}: " . (%oParamHash ? "remote (" .
$self->{oRemote}->command_param_string(\%oParamHash) . ") :" : "") . $strDebug;
&log(DEBUG, $strDebug);
if (%oParamHash)
{
my $strRemote = "remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . ")";
$strDebug = "${strOperation}: ${strRemote}: ${strDebug}";
&log(TRACE, "${strOperation}: ${strRemote}");
}
# If an operation is defined then write it
if (%oParamHash)
@@ -904,15 +818,45 @@ sub copy
# If this is the controlling process then wait for OK from remote
if (%oParamHash)
{
$self->{oRemote}->output_read(false, $strDebug);
# Test for an error when reading output
my $strOutput;
eval
{
$strOutput = $self->{oRemote}->output_read($strOperation eq OP_FILE_COPY, $strDebug);
};
# If there is an error then evaluate
if ($@)
{
my $oMessage = $@;
# We'll ignore this error if the source file was missing and missing file exception was returned
# and bIgnoreMissingSource is set
if ($bIgnoreMissingSource && $strRemote eq "in" && $oMessage->isa("BackRest::Exception") &&
$oMessage->code() == COMMAND_ERR_FILE_MISSING)
{
close($hDestinationFile) or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
unlink($strDestinationTmpOp) or confess &log(ERROR, "cannot remove file ${strDestinationTmpOp}");
return false;
}
# Otherwise report the error
confess $oMessage;
}
# If this was a remote copy, then return the result
if ($strOperation eq OP_FILE_COPY)
{
return false; #$strOutput eq 'N' ? true : false;
}
}
}
# Else this is a local operation
else
{
$strDebug = OP_FILE_COPY . ": $strDebug";
&log(DEBUG, $strDebug);
# If the source is compressed and the destination is not then decompress
if ($bSourceCompressed && !$bDestinationCompress)
{

307
test/lib/BackRestTest/FileTest.pm
View File

@@ -790,162 +790,175 @@ sub BackRestFileTest
{
$iRun = 0;
# Loop through backup local vs remote
for (my $bBackupRemote = 0; $bBackupRemote <= 1; $bBackupRemote++)
{
# Loop through source compression
for (my $bDbRemote = 0; $bDbRemote <= 1; $bDbRemote++)
# Loop through db local vs remote
for (my $bDbRemote = 0; $bDbRemote <= 1; $bDbRemote++)
{
# Backup and db cannot both be remote
if ($bBackupRemote && $bDbRemote)
{
# Backup and db cannot both be remote
if ($bBackupRemote && $bDbRemote)
next;
}
# Determine side is remote
my $strRemote = $bBackupRemote ? 'backup' : $bDbRemote ? 'db' : undef;
# Create the file object
my $oFile = BackRest::File->new
(
strStanza => "db",
strBackupClusterPath => undef,
strBackupPath => ${strTestPath},
strRemote => $strRemote,
oRemote => $bBackupRemote || $bDbRemote ? $oRemote : undef
);
# Loop through source compression
for (my $bSourceCompressed = 0; $bSourceCompressed <= 1; $bSourceCompressed++)
{
# Loop through destination compression
for (my $bDestinationCompress = 0; $bDestinationCompress <= 1; $bDestinationCompress++)
{
# Loop through source path types
for (my $bSourcePathType = 0; $bSourcePathType <= 1; $bSourcePathType++)
{
# Loop through destination path types
for (my $bDestinationPathType = 0; $bDestinationPathType <= 1; $bDestinationPathType++)
{
# Loop through source missing/present
for (my $bSourceMissing = 0; $bSourceMissing <= 1; $bSourceMissing++)
{
# Loop through source ignore/require
for (my $bSourceIgnoreMissing = 0; $bSourceIgnoreMissing <= 1; $bSourceIgnoreMissing++)
{
my $strSourcePathType = $bSourcePathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
my $strSourcePath = $bSourcePathType ? "db" : "backup";
my $strDestinationPathType = $bDestinationPathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
my $strDestinationPath = $bDestinationPathType ? "db" : "backup";
$iRun++;
# if ($iRun != 80)
# {
# next;
# }
&log(INFO, "run ${iRun} - " .
"srcpth " . (defined($strRemote) && $strRemote eq $strSourcePath ? "remote" : "local") .
":${strSourcePath}, srccmp $bSourceCompressed, srcmiss ${bSourceMissing}, " .
"srcignmiss ${bSourceIgnoreMissing}, " .
"dstpth " . (defined($strRemote) && $strRemote eq $strDestinationPath ? "remote" : "local") .
":${strDestinationPath}, dstcmp $bDestinationCompress");
# Drop the old test directory and create a new one
system("rm -rf test");
system("mkdir -p test/lock") == 0 or confess "Unable to create test/lock directory";
system("mkdir -p test/backup") == 0 or confess "Unable to create test/backup directory";
system("mkdir -p test/db") == 0 or confess "Unable to create test/db directory";
my $strSourceFile = "${strTestPath}/${strSourcePath}/test-source.txt";
my $strDestinationFile = "${strTestPath}/${strDestinationPath}/test-destination.txt";
# Create the compressed or uncompressed test file
my $strSourceHash;
if (!$bSourceMissing)
{
next;
}
system("echo 'TESTDATA' > ${strSourceFile}");
$strSourceHash = $oFile->hash(PATH_ABSOLUTE, $strSourceFile);
my $strRemote = $bBackupRemote ? 'backup' : $bDbRemote ? 'db' : undef;
my $oFile = BackRest::File->new
(
strStanza => "db",
strBackupClusterPath => undef,
strBackupPath => ${strTestPath},
strRemote => $strRemote,
oRemote => $bBackupRemote || $bDbRemote ? $oRemote : undef
);
for (my $bSourceCompressed = 0; $bSourceCompressed <= 1; $bSourceCompressed++)
{
# Loop through destination compression
for (my $bDestinationCompress = 0; $bDestinationCompress <= 1; $bDestinationCompress++)
if ($bSourceCompressed)
{
for (my $bSourcePathType = 0; $bSourcePathType <= 1; $bSourcePathType++)
{
my $strSourcePathType = $bSourcePathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
my $strSourcePath = $bSourcePathType ? "db" : "backup";
for (my $bDestinationPathType = 0; $bDestinationPathType <= 1; $bDestinationPathType++)
{
for (my $bSourceMissing = 0; $bSourceMissing <= 1; $bSourceMissing++)
{
my $strDestinationPathType = $bDestinationPathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
my $strDestinationPath = $bDestinationPathType ? "db" : "backup";
$iRun++;
# if ($iRun != 27)
# {
# next;
# }
&log(INFO, "run ${iRun} - " .
"srcpth " . (defined($strRemote) && $strRemote eq $strSourcePath ? "remote" : "local") .
":${strSourcePath}, srccmp $bSourceCompressed, srcmiss ${bSourceMissing}, " .
"dstpth " . (defined($strRemote) && $strRemote eq $strDestinationPath ? "remote" : "local") .
":${strDestinationPath}, dstcmp $bDestinationCompress");
# Drop the old test directory and create a new one
system("rm -rf test");
system("mkdir -p test/lock") == 0 or confess "Unable to create test/lock directory";
system("mkdir -p test/backup") == 0 or confess "Unable to create test/backup directory";
system("mkdir -p test/db") == 0 or confess "Unable to create test/db directory";
my $strSourceFile = "${strTestPath}/${strSourcePath}/test-source.txt";
my $strDestinationFile = "${strTestPath}/${strDestinationPath}/test-destination.txt";
# Create the compressed or uncompressed test file
my $strSourceHash;
if (!$bSourceMissing)
{
system("echo 'TESTDATA' > ${strSourceFile}");
$strSourceHash = $oFile->hash(PATH_ABSOLUTE, $strSourceFile);
if ($bSourceCompressed)
{
system("gzip ${strSourceFile}");
$strSourceFile .= ".gz";
}
}
if ($bDestinationCompress)
{
$strDestinationFile .= ".gz";
}
# Run file copy in an eval block because some errors are expected
my $bReturn;
eval
{
$bReturn = $oFile->copy($strSourcePathType, $strSourceFile,
$strDestinationPathType, $strDestinationFile,
$bSourceCompressed, $bDestinationCompress);
};
# Check for errors after copy
if ($@)
{
my $oMessage = $@;
if (blessed($oMessage))
{
if ($oMessage->isa("BackRest::Exception"))
{
if ($bSourceMissing)
{
next;
}
confess $oMessage->message();
}
else
{
confess 'unknown error object: ' . $oMessage;
}
}
else
{
confess $oMessage;
}
}
if ($bReturn)
{
if ($bSourceMissing)
{
confess "expected source file missing error";
}
# my $strDestinationFileCheck = $strDestinationFile;
#
# # Check for errors after copy
# if ($bDestinationCompress)
# {
# $strDestinationFileCheck .= ".gz";
# }
unless (-e $strDestinationFile)
{
confess "could not find destination file ${strDestinationFile}";
}
if ($bDestinationCompress)
{
system("gzip -d ${strDestinationFile}") == 0 or die "could not decompress ${strDestinationFile}";
$strDestinationFile = substr($strDestinationFile, 0, length($strDestinationFile) - 3);
}
my $strDestinationHash = $oFile->hash(PATH_ABSOLUTE, $strDestinationFile);
if ($strSourceHash ne $strDestinationHash)
{
confess "source ${strSourceHash} and destination ${strDestinationHash} file hashes do not match";
}
}
}
}
}
system("gzip ${strSourceFile}");
$strSourceFile .= ".gz";
}
}
if ($bDestinationCompress)
{
$strDestinationFile .= ".gz";
}
# Run file copy in an eval block because some errors are expected
my $bReturn;
eval
{
$bReturn = $oFile->copy($strSourcePathType, $strSourceFile,
$strDestinationPathType, $strDestinationFile,
$bSourceCompressed, $bDestinationCompress,
$bSourceIgnoreMissing);
};
# Check for errors after copy
if ($@)
{
my $oMessage = $@;
if (blessed($oMessage))
{
if ($oMessage->isa("BackRest::Exception"))
{
if ($bSourceMissing && !$bSourceIgnoreMissing)
{
next;
}
confess $oMessage->message();
}
else
{
confess 'unknown error object: ' . $oMessage;
}
}
else
{
confess $oMessage;
}
}
if ($bSourceMissing)
{
if ($bSourceIgnoreMissing)
{
if ($bReturn)
{
confess 'copy() returned ' . $bReturn . ' when ignore missing set';
}
next;
}
confess "expected source file missing error";
}
unless (-e $strDestinationFile)
{
confess "could not find destination file ${strDestinationFile}";
}
if ($bDestinationCompress)
{
system("gzip -d ${strDestinationFile}") == 0 or die "could not decompress ${strDestinationFile}";
$strDestinationFile = substr($strDestinationFile, 0, length($strDestinationFile) - 3);
}
my $strDestinationHash = $oFile->hash(PATH_ABSOLUTE, $strDestinationFile);
if ($strSourceHash ne $strDestinationHash)
{
confess "source ${strSourceHash} and destination ${strDestinationHash} file hashes do not match";
}
}
}
}
}
}
}
}
}
}
}