mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2024-12-14 10:13:05 +02:00
Ignore missing source in copy() working.
This commit is contained in:
David Steele
parent
e1903d53d0
commit
80206a28e9
@ -113,12 +113,12 @@ while ($strCommand ne OP_EXIT)
|
||||
}
|
||||
elsif ($strCommand eq OP_FILE_COPY)
|
||||
{
|
||||
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
|
||||
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
|
||||
param_get(\%oParamHash, 'source_compressed'),
|
||||
param_get(\%oParamHash, 'destination_compress'));
|
||||
|
||||
$oRemote->output_write();
|
||||
$oRemote->output_write(
|
||||
$oFile->copy(PATH_ABSOLUTE, param_get(\%oParamHash, 'source_file'),
|
||||
PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
|
||||
param_get(\%oParamHash, 'source_compressed'),
|
||||
param_get(\%oParamHash, 'destination_compress'),
|
||||
param_get(\%oParamHash, 'ignore_missing_source', false)) ? 'Y' : 'N');
|
||||
}
|
||||
elsif ($strCommand eq OP_FILE_COPY_IN)
|
||||
{
|
||||
|
||||
@ -569,111 +569,6 @@ sub move
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# PIPE_TO_STRING Function
|
||||
#
|
||||
# Copies data from a file handle into a string.
|
||||
####################################################################################################################################
|
||||
# sub pipe_to_string
|
||||
# {
|
||||
# my $self = shift;
|
||||
# my $hOut = shift;
|
||||
#
|
||||
# my $strBuffer;
|
||||
# my $hString = IO::String->new($strBuffer);
|
||||
# $self->pipe($hOut, $hString);
|
||||
#
|
||||
# return $strBuffer;
|
||||
# }
|
||||
|
||||
####################################################################################################################################
|
||||
# PIPE Function
|
||||
#
|
||||
# Copies data from one file handle to another, optionally compressing or decompressing the data in stream.
|
||||
####################################################################################################################################
|
||||
# sub pipe
|
||||
# {
|
||||
# my $self = shift;
|
||||
# my $hIn = shift;
|
||||
# my $hOut = shift;
|
||||
# my $bCompress = shift;
|
||||
# my $bUncompress = shift;
|
||||
#
|
||||
# # If compression is requested and the file is not already compressed
|
||||
# if (defined($bCompress) && $bCompress)
|
||||
# {
|
||||
# if (!gzip($hIn => $hOut))
|
||||
# {
|
||||
# confess $GzipError;
|
||||
# }
|
||||
# }
|
||||
# # If no compression is requested and the file is already compressed
|
||||
# elsif (defined($bUncompress) && $bUncompress)
|
||||
# {
|
||||
# if (!gunzip($hIn => $hOut))
|
||||
# {
|
||||
# confess $GunzipError;
|
||||
# }
|
||||
# }
|
||||
# # Else it's a straight copy
|
||||
# else
|
||||
# {
|
||||
# my $strBuffer;
|
||||
# my $iResultRead;
|
||||
# my $iResultWrite;
|
||||
#
|
||||
# # Read from the input handle
|
||||
# while (($iResultRead = sysread($hIn, $strBuffer, BLOCK_SIZE)) != 0)
|
||||
# {
|
||||
# if (!defined($iResultRead))
|
||||
# {
|
||||
# confess $!;
|
||||
# last;
|
||||
# }
|
||||
# else
|
||||
# {
|
||||
# # Write to the output handle
|
||||
# $iResultWrite = syswrite($hOut, $strBuffer, $iResultRead);
|
||||
#
|
||||
# if (!defined($iResultWrite) || $iResultWrite != $iResultRead)
|
||||
# {
|
||||
# confess $!;
|
||||
# last;
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
|
||||
####################################################################################################################################
|
||||
# WAIT_PID
|
||||
####################################################################################################################################
|
||||
# sub wait_pid
|
||||
# {
|
||||
# my $self = shift;
|
||||
# my $pId = shift;
|
||||
# my $strCommand = shift;
|
||||
# my $hIn = shift;
|
||||
# my $hOut = shift;
|
||||
# my $hErr = shift;
|
||||
#
|
||||
# # Close hIn
|
||||
# close($hIn);
|
||||
#
|
||||
# # Read STDERR into a string
|
||||
# my $strError = defined($hErr) ? $self->pipe_to_string($hErr) : "[unknown]";
|
||||
#
|
||||
# # Wait for the process to finish and report any errors
|
||||
# waitpid($pId, 0);
|
||||
# my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
|
||||
#
|
||||
# if ($iExitStatus != 0)
|
||||
# {
|
||||
# confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus . ": " .
|
||||
# (defined($strError) ? $strError : "[unknown]"));
|
||||
# }
|
||||
# }
|
||||
|
||||
####################################################################################################################################
|
||||
# EXISTS - Checks for the existence of a file, but does not imply that the file is readable/writeable.
|
||||
#
|
||||
@ -780,14 +675,15 @@ sub copy
|
||||
my $strDestinationTmpOp = $strDestinationPathType eq PIPE_STDOUT ?
|
||||
undef : $self->path_get($strDestinationPathType, $strDestinationFile, true);
|
||||
|
||||
# Set operation and debug string
|
||||
# my $strOperation = "File->copy[unknown]";
|
||||
# Set debug string and log
|
||||
my $strDebug = ($bSourceRemote ? " remote" : " local") . " ${strSourcePathType}" .
|
||||
(defined($strSourceFile) ? ":${strSourceFile}" : "") .
|
||||
" to" . ($bDestinationRemote ? " remote" : " local") . " ${strDestinationPathType}" .
|
||||
(defined($strDestinationFile) ? ":${strDestinationFile}" : "") .
|
||||
", source_compressed = " . ($bSourceCompressed ? "true" : "false") .
|
||||
", destination_compress = " . ($bDestinationCompress ? "true" : "false");
|
||||
", destination_compress = " . ($bDestinationCompress ? "true" : "false") .
|
||||
", ignore_missing_source = " . ($bIgnoreMissingSource ? "true" : "false");
|
||||
&log(DEBUG, OP_FILE_COPY . ": ${strDebug}");
|
||||
|
||||
# Open the source and destination files (if needed)
|
||||
my $hSourceFile;
|
||||
@ -804,13 +700,22 @@ sub copy
|
||||
{
|
||||
# $strError = 'file is missing';
|
||||
$iErrorCode = COMMAND_ERR_FILE_MISSING;
|
||||
|
||||
if ($bIgnoreMissingSource && $strDestinationPathType ne PIPE_STDOUT)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
$strError = "cannot open source file ${strSourceOp}: " . $strError;
|
||||
|
||||
if ($strSourcePathType eq PATH_ABSOLUTE)
|
||||
{
|
||||
$self->{oRemote}->write_line(*STDOUT, "block 0");
|
||||
if ($strDestinationPathType eq PIPE_STDOUT)
|
||||
{
|
||||
$self->{oRemote}->write_line(*STDOUT, "block 0");
|
||||
}
|
||||
|
||||
confess &log(ERROR, $strError, $iErrorCode);
|
||||
}
|
||||
|
||||
@ -882,12 +787,21 @@ sub copy
|
||||
$oParamHash{source_compressed} = $bSourceCompressed;
|
||||
$oParamHash{destination_file} = $strDestinationOp;
|
||||
$oParamHash{destination_compress} = $bDestinationCompress;
|
||||
|
||||
if ($bIgnoreMissingSource)
|
||||
{
|
||||
$oParamHash{ignore_missing_source} = $bIgnoreMissingSource;
|
||||
}
|
||||
}
|
||||
|
||||
# Build debug string
|
||||
$strDebug = "${strOperation}: " . (%oParamHash ? "remote (" .
|
||||
$self->{oRemote}->command_param_string(\%oParamHash) . ") :" : "") . $strDebug;
|
||||
&log(DEBUG, $strDebug);
|
||||
if (%oParamHash)
|
||||
{
|
||||
my $strRemote = "remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . ")";
|
||||
$strDebug = "${strOperation}: ${strRemote}: ${strDebug}";
|
||||
|
||||
&log(TRACE, "${strOperation}: ${strRemote}");
|
||||
}
|
||||
|
||||
# If an operation is defined then write it
|
||||
if (%oParamHash)
|
||||
@ -904,15 +818,45 @@ sub copy
|
||||
# If this is the controlling process then wait for OK from remote
|
||||
if (%oParamHash)
|
||||
{
|
||||
$self->{oRemote}->output_read(false, $strDebug);
|
||||
# Test for an error when reading output
|
||||
my $strOutput;
|
||||
|
||||
eval
|
||||
{
|
||||
$strOutput = $self->{oRemote}->output_read($strOperation eq OP_FILE_COPY, $strDebug);
|
||||
|
||||
};
|
||||
|
||||
# If there is an error then evaluate
|
||||
if ($@)
|
||||
{
|
||||
my $oMessage = $@;
|
||||
|
||||
# We'll ignore this error if the source file was missing and missing file exception was returned
|
||||
# and bIgnoreMissingSource is set
|
||||
if ($bIgnoreMissingSource && $strRemote eq "in" && $oMessage->isa("BackRest::Exception") &&
|
||||
$oMessage->code() == COMMAND_ERR_FILE_MISSING)
|
||||
{
|
||||
close($hDestinationFile) or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
|
||||
unlink($strDestinationTmpOp) or confess &log(ERROR, "cannot remove file ${strDestinationTmpOp}");
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
# Otherwise report the error
|
||||
confess $oMessage;
|
||||
}
|
||||
|
||||
# If this was a remote copy, then return the result
|
||||
if ($strOperation eq OP_FILE_COPY)
|
||||
{
|
||||
return false; #$strOutput eq 'N' ? true : false;
|
||||
}
|
||||
}
|
||||
}
|
||||
# Else this is a local operation
|
||||
else
|
||||
{
|
||||
$strDebug = OP_FILE_COPY . ": $strDebug";
|
||||
&log(DEBUG, $strDebug);
|
||||
|
||||
# If the source is compressed and the destination is not then decompress
|
||||
if ($bSourceCompressed && !$bDestinationCompress)
|
||||
{
|
||||
|
||||
@ -790,162 +790,175 @@ sub BackRestFileTest
|
||||
{
|
||||
$iRun = 0;
|
||||
|
||||
# Loop through backup local vs remote
|
||||
for (my $bBackupRemote = 0; $bBackupRemote <= 1; $bBackupRemote++)
|
||||
{
|
||||
# Loop through source compression
|
||||
for (my $bDbRemote = 0; $bDbRemote <= 1; $bDbRemote++)
|
||||
# Loop through db local vs remote
|
||||
for (my $bDbRemote = 0; $bDbRemote <= 1; $bDbRemote++)
|
||||
{
|
||||
# Backup and db cannot both be remote
|
||||
if ($bBackupRemote && $bDbRemote)
|
||||
{
|
||||
# Backup and db cannot both be remote
|
||||
if ($bBackupRemote && $bDbRemote)
|
||||
next;
|
||||
}
|
||||
|
||||
# Determine side is remote
|
||||
my $strRemote = $bBackupRemote ? 'backup' : $bDbRemote ? 'db' : undef;
|
||||
|
||||
# Create the file object
|
||||
my $oFile = BackRest::File->new
|
||||
(
|
||||
strStanza => "db",
|
||||
strBackupClusterPath => undef,
|
||||
strBackupPath => ${strTestPath},
|
||||
strRemote => $strRemote,
|
||||
oRemote => $bBackupRemote || $bDbRemote ? $oRemote : undef
|
||||
);
|
||||
|
||||
# Loop through source compression
|
||||
for (my $bSourceCompressed = 0; $bSourceCompressed <= 1; $bSourceCompressed++)
|
||||
{
|
||||
# Loop through destination compression
|
||||
for (my $bDestinationCompress = 0; $bDestinationCompress <= 1; $bDestinationCompress++)
|
||||
{
|
||||
# Loop through source path types
|
||||
for (my $bSourcePathType = 0; $bSourcePathType <= 1; $bSourcePathType++)
|
||||
{
|
||||
# Loop through destination path types
|
||||
for (my $bDestinationPathType = 0; $bDestinationPathType <= 1; $bDestinationPathType++)
|
||||
{
|
||||
# Loop through source missing/present
|
||||
for (my $bSourceMissing = 0; $bSourceMissing <= 1; $bSourceMissing++)
|
||||
{
|
||||
# Loop through source ignore/require
|
||||
for (my $bSourceIgnoreMissing = 0; $bSourceIgnoreMissing <= 1; $bSourceIgnoreMissing++)
|
||||
{
|
||||
my $strSourcePathType = $bSourcePathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
|
||||
my $strSourcePath = $bSourcePathType ? "db" : "backup";
|
||||
|
||||
my $strDestinationPathType = $bDestinationPathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
|
||||
my $strDestinationPath = $bDestinationPathType ? "db" : "backup";
|
||||
|
||||
$iRun++;
|
||||
|
||||
# if ($iRun != 80)
|
||||
# {
|
||||
# next;
|
||||
# }
|
||||
|
||||
&log(INFO, "run ${iRun} - " .
|
||||
"srcpth " . (defined($strRemote) && $strRemote eq $strSourcePath ? "remote" : "local") .
|
||||
":${strSourcePath}, srccmp $bSourceCompressed, srcmiss ${bSourceMissing}, " .
|
||||
"srcignmiss ${bSourceIgnoreMissing}, " .
|
||||
"dstpth " . (defined($strRemote) && $strRemote eq $strDestinationPath ? "remote" : "local") .
|
||||
":${strDestinationPath}, dstcmp $bDestinationCompress");
|
||||
|
||||
# Drop the old test directory and create a new one
|
||||
system("rm -rf test");
|
||||
system("mkdir -p test/lock") == 0 or confess "Unable to create test/lock directory";
|
||||
system("mkdir -p test/backup") == 0 or confess "Unable to create test/backup directory";
|
||||
system("mkdir -p test/db") == 0 or confess "Unable to create test/db directory";
|
||||
|
||||
my $strSourceFile = "${strTestPath}/${strSourcePath}/test-source.txt";
|
||||
my $strDestinationFile = "${strTestPath}/${strDestinationPath}/test-destination.txt";
|
||||
|
||||
# Create the compressed or uncompressed test file
|
||||
my $strSourceHash;
|
||||
|
||||
if (!$bSourceMissing)
|
||||
{
|
||||
next;
|
||||
}
|
||||
system("echo 'TESTDATA' > ${strSourceFile}");
|
||||
$strSourceHash = $oFile->hash(PATH_ABSOLUTE, $strSourceFile);
|
||||
|
||||
my $strRemote = $bBackupRemote ? 'backup' : $bDbRemote ? 'db' : undef;
|
||||
|
||||
my $oFile = BackRest::File->new
|
||||
(
|
||||
strStanza => "db",
|
||||
strBackupClusterPath => undef,
|
||||
strBackupPath => ${strTestPath},
|
||||
strRemote => $strRemote,
|
||||
oRemote => $bBackupRemote || $bDbRemote ? $oRemote : undef
|
||||
);
|
||||
|
||||
for (my $bSourceCompressed = 0; $bSourceCompressed <= 1; $bSourceCompressed++)
|
||||
{
|
||||
# Loop through destination compression
|
||||
for (my $bDestinationCompress = 0; $bDestinationCompress <= 1; $bDestinationCompress++)
|
||||
if ($bSourceCompressed)
|
||||
{
|
||||
for (my $bSourcePathType = 0; $bSourcePathType <= 1; $bSourcePathType++)
|
||||
{
|
||||
my $strSourcePathType = $bSourcePathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
|
||||
my $strSourcePath = $bSourcePathType ? "db" : "backup";
|
||||
|
||||
for (my $bDestinationPathType = 0; $bDestinationPathType <= 1; $bDestinationPathType++)
|
||||
{
|
||||
for (my $bSourceMissing = 0; $bSourceMissing <= 1; $bSourceMissing++)
|
||||
{
|
||||
my $strDestinationPathType = $bDestinationPathType ? PATH_DB_ABSOLUTE : PATH_BACKUP_ABSOLUTE;
|
||||
my $strDestinationPath = $bDestinationPathType ? "db" : "backup";
|
||||
|
||||
$iRun++;
|
||||
|
||||
# if ($iRun != 27)
|
||||
# {
|
||||
# next;
|
||||
# }
|
||||
|
||||
&log(INFO, "run ${iRun} - " .
|
||||
"srcpth " . (defined($strRemote) && $strRemote eq $strSourcePath ? "remote" : "local") .
|
||||
":${strSourcePath}, srccmp $bSourceCompressed, srcmiss ${bSourceMissing}, " .
|
||||
"dstpth " . (defined($strRemote) && $strRemote eq $strDestinationPath ? "remote" : "local") .
|
||||
":${strDestinationPath}, dstcmp $bDestinationCompress");
|
||||
|
||||
# Drop the old test directory and create a new one
|
||||
system("rm -rf test");
|
||||
system("mkdir -p test/lock") == 0 or confess "Unable to create test/lock directory";
|
||||
system("mkdir -p test/backup") == 0 or confess "Unable to create test/backup directory";
|
||||
system("mkdir -p test/db") == 0 or confess "Unable to create test/db directory";
|
||||
|
||||
my $strSourceFile = "${strTestPath}/${strSourcePath}/test-source.txt";
|
||||
my $strDestinationFile = "${strTestPath}/${strDestinationPath}/test-destination.txt";
|
||||
|
||||
# Create the compressed or uncompressed test file
|
||||
my $strSourceHash;
|
||||
|
||||
if (!$bSourceMissing)
|
||||
{
|
||||
system("echo 'TESTDATA' > ${strSourceFile}");
|
||||
$strSourceHash = $oFile->hash(PATH_ABSOLUTE, $strSourceFile);
|
||||
|
||||
if ($bSourceCompressed)
|
||||
{
|
||||
system("gzip ${strSourceFile}");
|
||||
$strSourceFile .= ".gz";
|
||||
}
|
||||
}
|
||||
|
||||
if ($bDestinationCompress)
|
||||
{
|
||||
$strDestinationFile .= ".gz";
|
||||
}
|
||||
|
||||
# Run file copy in an eval block because some errors are expected
|
||||
my $bReturn;
|
||||
|
||||
eval
|
||||
{
|
||||
$bReturn = $oFile->copy($strSourcePathType, $strSourceFile,
|
||||
$strDestinationPathType, $strDestinationFile,
|
||||
$bSourceCompressed, $bDestinationCompress);
|
||||
};
|
||||
|
||||
# Check for errors after copy
|
||||
if ($@)
|
||||
{
|
||||
my $oMessage = $@;
|
||||
|
||||
if (blessed($oMessage))
|
||||
{
|
||||
if ($oMessage->isa("BackRest::Exception"))
|
||||
{
|
||||
if ($bSourceMissing)
|
||||
{
|
||||
next;
|
||||
}
|
||||
|
||||
confess $oMessage->message();
|
||||
}
|
||||
else
|
||||
{
|
||||
confess 'unknown error object: ' . $oMessage;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
confess $oMessage;
|
||||
}
|
||||
}
|
||||
|
||||
if ($bReturn)
|
||||
{
|
||||
if ($bSourceMissing)
|
||||
{
|
||||
confess "expected source file missing error";
|
||||
}
|
||||
# my $strDestinationFileCheck = $strDestinationFile;
|
||||
#
|
||||
# # Check for errors after copy
|
||||
# if ($bDestinationCompress)
|
||||
# {
|
||||
# $strDestinationFileCheck .= ".gz";
|
||||
# }
|
||||
|
||||
unless (-e $strDestinationFile)
|
||||
{
|
||||
confess "could not find destination file ${strDestinationFile}";
|
||||
}
|
||||
|
||||
if ($bDestinationCompress)
|
||||
{
|
||||
system("gzip -d ${strDestinationFile}") == 0 or die "could not decompress ${strDestinationFile}";
|
||||
$strDestinationFile = substr($strDestinationFile, 0, length($strDestinationFile) - 3);
|
||||
}
|
||||
|
||||
my $strDestinationHash = $oFile->hash(PATH_ABSOLUTE, $strDestinationFile);
|
||||
|
||||
if ($strSourceHash ne $strDestinationHash)
|
||||
{
|
||||
confess "source ${strSourceHash} and destination ${strDestinationHash} file hashes do not match";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
system("gzip ${strSourceFile}");
|
||||
$strSourceFile .= ".gz";
|
||||
}
|
||||
}
|
||||
|
||||
if ($bDestinationCompress)
|
||||
{
|
||||
$strDestinationFile .= ".gz";
|
||||
}
|
||||
|
||||
# Run file copy in an eval block because some errors are expected
|
||||
my $bReturn;
|
||||
|
||||
eval
|
||||
{
|
||||
$bReturn = $oFile->copy($strSourcePathType, $strSourceFile,
|
||||
$strDestinationPathType, $strDestinationFile,
|
||||
$bSourceCompressed, $bDestinationCompress,
|
||||
$bSourceIgnoreMissing);
|
||||
};
|
||||
|
||||
# Check for errors after copy
|
||||
if ($@)
|
||||
{
|
||||
my $oMessage = $@;
|
||||
|
||||
if (blessed($oMessage))
|
||||
{
|
||||
if ($oMessage->isa("BackRest::Exception"))
|
||||
{
|
||||
if ($bSourceMissing && !$bSourceIgnoreMissing)
|
||||
{
|
||||
next;
|
||||
}
|
||||
|
||||
confess $oMessage->message();
|
||||
}
|
||||
else
|
||||
{
|
||||
confess 'unknown error object: ' . $oMessage;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
confess $oMessage;
|
||||
}
|
||||
}
|
||||
|
||||
if ($bSourceMissing)
|
||||
{
|
||||
if ($bSourceIgnoreMissing)
|
||||
{
|
||||
if ($bReturn)
|
||||
{
|
||||
confess 'copy() returned ' . $bReturn . ' when ignore missing set';
|
||||
}
|
||||
|
||||
next;
|
||||
}
|
||||
|
||||
confess "expected source file missing error";
|
||||
}
|
||||
|
||||
unless (-e $strDestinationFile)
|
||||
{
|
||||
confess "could not find destination file ${strDestinationFile}";
|
||||
}
|
||||
|
||||
if ($bDestinationCompress)
|
||||
{
|
||||
system("gzip -d ${strDestinationFile}") == 0 or die "could not decompress ${strDestinationFile}";
|
||||
$strDestinationFile = substr($strDestinationFile, 0, length($strDestinationFile) - 3);
|
||||
}
|
||||
|
||||
my $strDestinationHash = $oFile->hash(PATH_ABSOLUTE, $strDestinationFile);
|
||||
|
||||
if ($strSourceHash ne $strDestinationHash)
|
||||
{
|
||||
confess "source ${strSourceHash} and destination ${strDestinationHash} file hashes do not match";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user