db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

More unit tests for file functions.

Browse Source
This commit is contained in:
David Steele 2014-05-27 09:00:24 -04:00
parent 3e12f9230b
commit 85f591f801
5 changed files with 200 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,3 +2,4 @@
*~
*.swp
test/test
test/lock

2
pg_backrest.conf
View File

@ -4,7 +4,7 @@ compress=/usr/bin/gzip --stdout %file%
decompress=/usr/bin/gzip -dc %file%
#checksum=sha1sum %file% | awk '{print $1}' # Ubuntu Linux
checksum=/usr/bin/shasum %file% | awk '{print $1}'
manifest=/opt/local/bin/gfind %path% -printf '%P\t%y\t%u\t%g\t%m\t%T@\t%i\t%s\t%l\n'
manifest=/opt/local/bin/gfind %path% -ignore_readdir_race -printf '%P\t%y\t%u\t%g\t%m\t%T@\t%i\t%s\t%l\n'
psql=/Library/PostgreSQL/9.3/bin/psql -X %option%
[global:log]

97
pg_backrest_file.pm
View File

@ -89,6 +89,11 @@ sub BUILD
# Create the ssh options string
if (defined($self->{strBackupHost}) || defined($self->{strDbHost}))
{
# if (defined($self->{strBackupHost}) && defined($self->{strDbHost}))
# {
# confess &log(ASSERT, "backup and db hosts cannot both be remote");
# }
my $strOptionSSHRequestTTY = "RequestTTY=yes";
my $strOptionSSHCompression = "Compression=no";
@ -103,7 +108,7 @@ sub BUILD
&log(TRACE, "connecting to backup ssh host " . $self->{strBackupHost});
$self->{oBackupSSH} = Net::OpenSSH->new($self->{strBackupHost}, timeout => 300, user => $self->{strBackupUser},
default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
# default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
master_opts => [-o => $strOptionSSHCompression, -o => $strOptionSSHRequestTTY]);
$self->{oBackupSSH}->error and confess &log(ERROR, "unable to connect to $self->{strBackupHost}: " . $self->{oBackupSSH}->error);
}
@ -114,7 +119,7 @@ sub BUILD
&log(TRACE, "connecting to database ssh host $self->{strDbHost}");
$self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, timeout => 300, user => $self->{strDbUser},
default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
# default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
master_opts => [-o => $strOptionSSHCompression, -o => $strOptionSSHRequestTTY]);
$self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error);
}
@ -171,6 +176,16 @@ sub error_get
return trim($strError);
}
####################################################################################################################################
# ERROR_CLEAR
####################################################################################################################################
sub error_clear
{
my $self = shift;
unlink($self->path_get(PATH_LOCK_ERR, "file"));
}
####################################################################################################################################
# PATH_GET
####################################################################################################################################
@ -251,7 +266,12 @@ sub path_get
# Get the lock error path
if ($strType eq PATH_LOCK_ERR)
{
my $strTempPath = "$self->{strLockPath}";
my $strTempPath = $self->{strLockPath};
if (!defined($strTempPath))
{
return undef;
}
return ${strTempPath} . (defined($strFile) ? "/${strFile}" .
(defined($self->{iThreadIdx}) ? ".$self->{iThreadIdx}" : "") . ".err" : "");
@ -631,11 +651,11 @@ sub file_copy
# Execute the command through ssh
my $oSSH = $self->remote_get($strSourcePathType);
unless ($oSSH->system({stdout_fh => $hFile}, $strCommand))
unless ($oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file"), stdout_fh => $hFile}, $strCommand))
{
close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}");
my $strResult = "unable to execute ssh '${strCommand}'";
my $strResult = "unable to execute ssh '${strCommand}': " . $self->error_get();
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
@ -647,21 +667,26 @@ sub file_copy
{
&log(TRACE, "file_copy: local ${strSource} ($strCommand) to remote ${strDestination}");
if (defined($self->path_get(PATH_LOCK_ERR, "file")))
{
$strCommand .= " 2> " . $self->path_get(PATH_LOCK_ERR, "file");
}
# Open the input command as a stream
my $hOut;
my $pId = open3(undef, $hOut, undef, $strCommand) or confess(ERROR, "unable to execute '${strCommand}'");
# Execute the command though ssh
my $oSSH = $self->remote_get($strDestinationPathType);
$oSSH->system({stdin_fh => $hOut}, "cat > ${strDestinationTmp}") or confess &log(ERROR, "unable to execute ssh 'cat'");
$oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file"), stdin_fh => $hOut}, "cat > ${strDestinationTmp}") or confess &log(ERROR, "unable to execute ssh 'cat'");
# Wait for the stream process to finish
waitpid($pId, 0);
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
if ($iExitStatus != 0)
{
my $strResult = "command '${strCommand}' returned " . $iExitStatus;
my $strResult = "command '${strCommand}' returned " . $iExitStatus . ": " . $self->error_get();
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
}
@ -684,7 +709,7 @@ sub file_copy
&log(TRACE, "file_copy: remote ${strSourcePathType} '${strCommand}'");
my $oSSH = $self->remote_get($strSourcePathType);
$oSSH->system($strCommand);
$oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file")}, $strCommand);
if ($oSSH->error)
{
@ -696,6 +721,11 @@ sub file_copy
{
&log(TRACE, "file_copy: local '${strCommand}'");
if (defined($self->path_get(PATH_LOCK_ERR, "file")))
{
$strCommand .= " 2> " . $self->path_get(PATH_LOCK_ERR, "file");
}
unless(system($strCommand) == 0)
{
my $strResult = "unable to copy local ${strSource} to local ${strDestinationTmp}";
@ -876,36 +906,55 @@ sub file_exists
# Get the root path for the manifest
my $strPathExists = $self->path_get($strPathType, $strPath);
# Builds the exists command
my $strCommand = "ls ${strPathExists}";
# Run the file exists command
my $strExists = "";
&log(TRACE, "file_exists: " . ($self->is_remote($strPathType) ? "remote" : "local") . " ${strPathType}:${strPathExists}");
# Run remotely
if ($self->is_remote($strPathType))
{
&log(TRACE, "file_exists: remote ${strPathType}:${strPathExists}");
# Builds the exists command
my $strCommand = "ls ${strPathExists}";
# Run the file exists command
my $strExists;
my $strError;
&log(TRACE, "file_exists: command: ${strCommand}");
my $oSSH = $self->remote_get($strPathType);
$strExists = trim($oSSH->capture($strCommand));
$strExists = $oSSH->capture({stderr_file => $self->path_get(PATH_LOCK_ERR, "file")}, $strCommand);
&log(TRACE, "file_exists: search = ${strPathExists}, result = " . (defined($strExists) ? $strExists : "<undef>"));
if ($oSSH->error)
{
confess &log(ERROR, "unable to execute file exists (${strCommand}): " . $self->error_get());
my $strError = $self->error_get();
&log(TRACE, "error detected: $strError");
if ($strError =~ /.*ls.*No such file or directory.*/)
{
return(false);
}
confess &log(ERROR, "unable to execute file exists (${strCommand}): " . $strError);
}
# If the return from ls eq strPathExists then true
return trim($strExists) eq $strPathExists;
}
# Run locally
else
{
&log(TRACE, "file_exists: local ${strPathType}:${strPathExists}");
$strExists = trim(capture($strCommand));
if (-e $strPathExists)
{
return true;
}
else
{
return false;
}
}
&log(TRACE, "file_exists: search = ${strPathExists}, result = ${strExists}");
# If the return from ls eq strPathExists then true
return ($strExists eq $strPathExists);
confess &log(ASSERT, "file_exists: true or false should have been returned");
}
####################################################################################################################################

5
pg_backrest_utility.pm
View File

@ -153,6 +153,11 @@ sub trim
{
my $strBuffer = shift;
if (!defined($strBuffer))
{
return undef;
}
$strBuffer =~ s/^\s+|\s+$//g;
return $strBuffer;

134
test/lib/BackRestTest/FileTest.pm
View File

@ -23,22 +23,109 @@ our @EXPORT = qw(BackRestTestFile);
sub BackRestTestFile
{
my $strLockPath = dirname(abs_path($0)) . "/lock";
my $strTestPath = dirname(abs_path($0)) . "/test";
my $iRun = 0;
log_level_set(OFF, OFF);
# Test copy()
for (my $bSourceRemote = 0; $bSourceRemote <= 1; $bSourceRemote++)
# !!! NEED TO TEST WHERE LOCK PATH IS UNDEF
# Test file_exists()
for (my $bRemote = 0; $bRemote <= 1; $bRemote++)
{
my $strSourceHost = $bSourceRemote ? "127.0.0.1" : undef;
my $strSourceUser = $bSourceRemote ? "dsteele" : undef;
my $strHost = $bRemote ? "127.0.0.1" : undef;
my $strUser = $bRemote ? "dsteele" : undef;
system("rm -rf lock");
system("mkdir -p lock") == 0 or confess "Unable to create lock directory";
my $oFile = pg_backrest_file->new
(
strStanza => "db",
bNoCompression => true,
strBackupClusterPath => ${strTestPath},
strBackupPath => ${strTestPath},
strBackupHost => $strHost,
strBackupUser => $strUser,
strLockPath => $strLockPath
);
# Loop through exists
for (my $bExists = 0; $bExists <= 1; $bExists++)
{
# Loop through error
for (my $bError = 0; $bError <= $bRemote ? 1 : 0; $bError++)
{
$iRun++;
print "run ${iRun} - " .
"rmt $bRemote, exist $bExists, error $bError\n";
# Drop the old test directory and create a new one
system("rm -rf test");
system("mkdir test") == 0 or confess "Unable to create test directory";
my $strFile = "${strTestPath}/test.txt";
if ($bExists)
{
system("echo 'TESTDATA' > ${strFile}");
}
if ($bError)
{
$strFile = "--backrest-error " . $strFile;
}
# Execute in eval in case of error
eval
{
if ($oFile->file_exists(PATH_BACKUP_ABSOLUTE, $strFile) != $bExists)
{
confess "bExists is set to $bExists, but file_exists() returned " . !$bExists;
}
};
if ($@)
{
if (!$bError)
{
confess 'error was returned but no error generated';
}
my $strError = $oFile->error_get();
if (!defined($strError) || ($strError eq ''))
{
confess 'no error message returned';
}
print " error raised: ${strError}\n";
next;
}
}
}
}
return;
# Test copy()
$iRun = 0;
system("rm -rf lock");
system("mkdir -p lock") == 0 or confess "Unable to create lock directory";
for (my $bBackupRemote = 0; $bBackupRemote <= 1; $bBackupRemote++)
{
my $strBackupHost = $bBackupRemote ? "127.0.0.1" : undef;
my $strBackupUser = $bBackupRemote ? "dsteele" : undef;
# Loop through source compression
for (my $bDestinationRemote = 0; $bDestinationRemote <= 1; $bDestinationRemote++)
for (my $bDbRemote = 0; $bDbRemote <= 1; $bDbRemote++)
{
my $strDestinationHost = $bDestinationRemote ? "127.0.0.1" : undef;
my $strDestinationUser = $bDestinationRemote ? "dsteele" : undef;
my $strDbHost = $bDbRemote ? "127.0.0.1" : undef;
my $strDbUser = $bDbRemote ? "dsteele" : undef;
# Loop through destination compression
for (my $bDestinationCompressed = 0; $bDestinationCompressed <= 1; $bDestinationCompressed++)
@ -49,10 +136,10 @@ sub BackRestTestFile
bNoCompression => !$bDestinationCompressed,
strBackupClusterPath => undef,
strBackupPath => ${strTestPath},
strBackupHost => $strSourceHost,
strBackupUser => $strSourceUser,
strDbHost => $strDestinationHost,
strDbUser => $strDestinationUser,
strBackupHost => $strBackupHost,
strBackupUser => $strBackupUser,
strDbHost => $strDbHost,
strDbUser => $strDbUser,
strCommandCompress => "gzip --stdout %file%",
strCommandDecompress => "gzip -dc %file%",
strLockPath => dirname($0) . "/test/lock"
@ -75,8 +162,8 @@ sub BackRestTestFile
$iRun++;
print "run ${iRun} - " .
"srcpth ${strSourcePath}, srcrmt $bSourceRemote, srccmp $bSourceCompressed, " .
"dstpth ${strDestinationPath}, dstrmt $bDestinationRemote, dstcmp $bDestinationCompressed, " .
"srcpth ${strSourcePath}, bkprmt $bBackupRemote, srccmp $bSourceCompressed, " .
"dstpth ${strDestinationPath}, dbrmt $bDbRemote, dstcmp $bDestinationCompressed, " .
"error $bError, confess_error $bConfessError\n";
# Drop the old test directory and create a new one
@ -118,13 +205,22 @@ sub BackRestTestFile
if ($@)
{
# Different remote and destination with different path types should error
if ($bSourceRemote && $bDestinationRemote)
if ($bBackupRemote && $bDbRemote && ($strSourcePath ne $strDestinationPath))
{
print " different source and remote for same path not supported\n";
next;
}
# If the error was intentional, then also continue
elsif ($bError)
{
my $strError = $oFile->error_get();
if (!defined($strError) || ($strError eq ''))
{
confess 'no error message returned';
}
print " error raised: ${strError}\n";
next;
}
# Else this is an unexpected error
@ -147,6 +243,14 @@ sub BackRestTestFile
}
else
{
my $strError = $oFile->error_get();
if (!defined($strError) || ($strError eq ''))
{
confess 'no error message returned';
}
print " error returned: ${strError}\n";
next;
}
}
@ -158,6 +262,8 @@ sub BackRestTestFile
{
confess "error was returned when no error generated";
}
print " true was returned\n";
}
# Check for errors after copy