More unit tests for file functions.

.gitignore

@@ -2,3 +2,4 @@
 *~
 *.swp
 test/test
+test/lock

pg_backrest.conf

@@ -4,7 +4,7 @@ compress=/usr/bin/gzip --stdout %file%
 decompress=/usr/bin/gzip -dc %file%
 #checksum=sha1sum %file% | awk '{print $1}' # Ubuntu Linux
 checksum=/usr/bin/shasum %file% | awk '{print $1}'
-manifest=/opt/local/bin/gfind %path% -printf '%P\t%y\t%u\t%g\t%m\t%T@\t%i\t%s\t%l\n'
+manifest=/opt/local/bin/gfind %path% -ignore_readdir_race -printf '%P\t%y\t%u\t%g\t%m\t%T@\t%i\t%s\t%l\n'
 psql=/Library/PostgreSQL/9.3/bin/psql -X %option%
 
 [global:log]

pg_backrest_file.pm

@@ -89,6 +89,11 @@ sub BUILD
     # Create the ssh options string
     if (defined($self->{strBackupHost}) || defined($self->{strDbHost}))
     {
+#        if (defined($self->{strBackupHost}) && defined($self->{strDbHost}))
+#        {
+#            confess &log(ASSERT, "backup and db hosts cannot both be remote");
+#        }
+        
         my $strOptionSSHRequestTTY = "RequestTTY=yes";
         my $strOptionSSHCompression = "Compression=no";
 
@@ -103,7 +108,7 @@ sub BUILD
             &log(TRACE, "connecting to backup ssh host " . $self->{strBackupHost});
             
             $self->{oBackupSSH} = Net::OpenSSH->new($self->{strBackupHost}, timeout => 300, user => $self->{strBackupUser},
-                                      default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
+#                                      default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
                                       master_opts => [-o => $strOptionSSHCompression, -o => $strOptionSSHRequestTTY]);
             $self->{oBackupSSH}->error and confess &log(ERROR, "unable to connect to $self->{strBackupHost}: " . $self->{oBackupSSH}->error);
         }
@@ -114,7 +119,7 @@ sub BUILD
             &log(TRACE, "connecting to database ssh host $self->{strDbHost}");
 
             $self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, timeout => 300, user => $self->{strDbUser},
-                                  default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
+#                                  default_stderr_file => $self->path_get(PATH_LOCK_ERR, "file"),
                                   master_opts => [-o => $strOptionSSHCompression, -o => $strOptionSSHRequestTTY]);
             $self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error);
         }
@@ -171,6 +176,16 @@ sub error_get
     return trim($strError);
 }
 
+####################################################################################################################################
+# ERROR_CLEAR
+####################################################################################################################################
+sub error_clear
+{
+    my $self = shift;
+
+    unlink($self->path_get(PATH_LOCK_ERR, "file"));
+}
+
 ####################################################################################################################################
 # PATH_GET
 ####################################################################################################################################
@@ -251,7 +266,12 @@ sub path_get
     # Get the lock error path
     if ($strType eq PATH_LOCK_ERR)
     {
-        my $strTempPath = "$self->{strLockPath}";
+        my $strTempPath = $self->{strLockPath};
+        
+        if (!defined($strTempPath))
+        {
+            return undef;
+        }
 
         return ${strTempPath} . (defined($strFile) ? "/${strFile}" .
                                 (defined($self->{iThreadIdx}) ? ".$self->{iThreadIdx}" : "") . ".err" : "");
@@ -631,11 +651,11 @@ sub file_copy
             # Execute the command through ssh
             my $oSSH = $self->remote_get($strSourcePathType);
             
-            unless ($oSSH->system({stdout_fh => $hFile}, $strCommand))
+            unless ($oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file"), stdout_fh => $hFile}, $strCommand))
             {
                 close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}");
                 
-                my $strResult = "unable to execute ssh '${strCommand}'";
+                my $strResult = "unable to execute ssh '${strCommand}': " . $self->error_get();
                 $bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
             }
 
@@ -647,13 +667,18 @@ sub file_copy
         {
             &log(TRACE, "file_copy: local ${strSource} ($strCommand) to remote ${strDestination}");
 
+            if (defined($self->path_get(PATH_LOCK_ERR, "file")))
+            {
+                $strCommand .= " 2> " . $self->path_get(PATH_LOCK_ERR, "file");
+            }
+
             # Open the input command as a stream
             my $hOut;
             my $pId = open3(undef, $hOut, undef, $strCommand) or confess(ERROR, "unable to execute '${strCommand}'");
 
             # Execute the command though ssh
             my $oSSH = $self->remote_get($strDestinationPathType);
-            $oSSH->system({stdin_fh => $hOut}, "cat > ${strDestinationTmp}") or confess &log(ERROR, "unable to execute ssh 'cat'");
+            $oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file"), stdin_fh => $hOut}, "cat > ${strDestinationTmp}") or confess &log(ERROR, "unable to execute ssh 'cat'");
 
             # Wait for the stream process to finish
             waitpid($pId, 0);
@@ -661,7 +686,7 @@ sub file_copy
 
             if ($iExitStatus != 0)
             {
-                my $strResult = "command '${strCommand}' returned " . $iExitStatus;
+                my $strResult = "command '${strCommand}' returned " . $iExitStatus . ": " . $self->error_get();
                 $bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
             }
         }
@@ -684,7 +709,7 @@ sub file_copy
             &log(TRACE, "file_copy: remote ${strSourcePathType} '${strCommand}'");
 
             my $oSSH = $self->remote_get($strSourcePathType);
-            $oSSH->system($strCommand);
+            $oSSH->system({stderr_file => $self->path_get(PATH_LOCK_ERR, "file")}, $strCommand);
 
             if ($oSSH->error)
             {
@@ -696,6 +721,11 @@ sub file_copy
         {
             &log(TRACE, "file_copy: local '${strCommand}'");
             
+            if (defined($self->path_get(PATH_LOCK_ERR, "file")))
+            {
+                $strCommand .= " 2> " . $self->path_get(PATH_LOCK_ERR, "file");
+            }
+
             unless(system($strCommand) == 0)
             {
                 my $strResult = "unable to copy local ${strSource} to local ${strDestinationTmp}";
@@ -876,36 +906,55 @@ sub file_exists
     # Get the root path for the manifest
     my $strPathExists = $self->path_get($strPathType, $strPath);
 
-    # Builds the exists command
+    &log(TRACE, "file_exists: " . ($self->is_remote($strPathType) ? "remote" : "local") . " ${strPathType}:${strPathExists}");
-    my $strCommand = "ls ${strPathExists}";
-    
-    # Run the file exists command
-    my $strExists = "";
 
     # Run remotely
     if ($self->is_remote($strPathType))
     {
-        &log(TRACE, "file_exists: remote ${strPathType}:${strPathExists}");
+        # Builds the exists command
+        my $strCommand = "ls ${strPathExists}";
+
+        # Run the file exists command
+        my $strExists;
+        my $strError;
+
+        &log(TRACE, "file_exists: command: ${strCommand}");
 
         my $oSSH = $self->remote_get($strPathType);
-        $strExists = trim($oSSH->capture($strCommand));
+        $strExists = $oSSH->capture({stderr_file => $self->path_get(PATH_LOCK_ERR, "file")}, $strCommand);
+
+        &log(TRACE, "file_exists: search = ${strPathExists}, result = " . (defined($strExists) ? $strExists : "<undef>"));
 
         if ($oSSH->error)
         {
-            confess &log(ERROR, "unable to execute file exists (${strCommand}): " . $self->error_get());
+            my $strError = $self->error_get();
+            &log(TRACE, "error detected: $strError");
+
+            if ($strError =~ /.*ls.*No such file or directory.*/)
+            {
+                return(false);
+            }
+
+            confess &log(ERROR, "unable to execute file exists (${strCommand}): " . $strError);
         }
+
+        # If the return from ls eq strPathExists then true
+        return trim($strExists) eq $strPathExists;
     }
     # Run locally
     else
     {
-        &log(TRACE, "file_exists: local ${strPathType}:${strPathExists}");
+        if (-e $strPathExists)
-        $strExists = trim(capture($strCommand));
+        {
+            return true;
+        }
+        else
+        {
+            return false;
+        }
     }
 
-    &log(TRACE, "file_exists: search = ${strPathExists}, result = ${strExists}");
+    confess &log(ASSERT, "file_exists: true or false should have been returned");
-
-    # If the return from ls eq strPathExists then true
-    return ($strExists eq $strPathExists);
 }
 
 ####################################################################################################################################

pg_backrest_utility.pm

@@ -153,6 +153,11 @@ sub trim
 {
     my $strBuffer = shift;
 
+    if (!defined($strBuffer))
+    {
+        return undef;
+    }
+
     $strBuffer =~ s/^\s+|\s+$//g;
 
     return $strBuffer;

test/lib/BackRestTest/FileTest.pm

@@ -23,22 +23,109 @@ our @EXPORT = qw(BackRestTestFile);
 
 sub BackRestTestFile
 {
+    my $strLockPath = dirname(abs_path($0)) . "/lock";
     my $strTestPath = dirname(abs_path($0)) . "/test";
     my $iRun = 0;
     
     log_level_set(OFF, OFF);
 
-    # Test copy()
+    # !!! NEED TO TEST WHERE LOCK PATH IS UNDEF
-    for (my $bSourceRemote = 0; $bSourceRemote <= 1; $bSourceRemote++)
+
+    # Test file_exists()
+    for (my $bRemote = 0; $bRemote <= 1; $bRemote++)
     {
-        my $strSourceHost = $bSourceRemote ? "127.0.0.1" : undef;
+        my $strHost = $bRemote ? "127.0.0.1" : undef;
-        my $strSourceUser = $bSourceRemote ? "dsteele" : undef;
+        my $strUser = $bRemote ? "dsteele" : undef;
+
+        system("rm -rf lock");
+        system("mkdir -p lock") == 0 or confess "Unable to create lock directory";
+
+        my $oFile = pg_backrest_file->new
+        (
+            strStanza => "db",
+            bNoCompression => true,
+            strBackupClusterPath => ${strTestPath},
+            strBackupPath => ${strTestPath},
+            strBackupHost => $strHost,
+            strBackupUser => $strUser,
+            strLockPath => $strLockPath
+        );
+
+        # Loop through exists
+        for (my $bExists = 0; $bExists <= 1; $bExists++)
+        {
+            # Loop through error
+            for (my $bError = 0; $bError <= $bRemote ? 1 : 0; $bError++)
+            {
+                $iRun++;
+
+                print "run ${iRun} - " .
+                      "rmt $bRemote, exist $bExists, error $bError\n";
+
+                # Drop the old test directory and create a new one
+                system("rm -rf test");
+                system("mkdir test") == 0 or confess "Unable to create test directory";
+
+                my $strFile = "${strTestPath}/test.txt";
+
+                if ($bExists)
+                {
+                    system("echo 'TESTDATA' > ${strFile}");
+                }
+
+                if ($bError)
+                {
+                    $strFile = "--backrest-error " . $strFile;
+                }
+                
+                # Execute in eval in case of error
+                eval
+                {
+                    if ($oFile->file_exists(PATH_BACKUP_ABSOLUTE, $strFile) != $bExists)
+                    {
+                        confess "bExists is set to $bExists, but file_exists() returned " . !$bExists;
+                    }
+                };
+                
+                if ($@)
+                {
+                    if (!$bError)
+                    {
+                        confess 'error was returned but no error generated';
+                    }
+                    
+                    my $strError = $oFile->error_get();
+                    
+                    if (!defined($strError) || ($strError eq ''))
+                    {
+                        confess 'no error message returned';
+                    }
+                    
+                    print "    error raised: ${strError}\n";
+                    next;
+                }
+            }
+        }
+    }
+
+    return;
+
+    # Test copy()
+    $iRun = 0;
+
+    system("rm -rf lock");
+    system("mkdir -p lock") == 0 or confess "Unable to create lock directory";
+
+    for (my $bBackupRemote = 0; $bBackupRemote <= 1; $bBackupRemote++)
+    {
+        my $strBackupHost = $bBackupRemote ? "127.0.0.1" : undef;
+        my $strBackupUser = $bBackupRemote ? "dsteele" : undef;
 
         # Loop through source compression
-        for (my $bDestinationRemote = 0; $bDestinationRemote <= 1; $bDestinationRemote++)
+        for (my $bDbRemote = 0; $bDbRemote <= 1; $bDbRemote++)
         {
-            my $strDestinationHost = $bDestinationRemote ? "127.0.0.1" : undef;
+            my $strDbHost = $bDbRemote ? "127.0.0.1" : undef;
-            my $strDestinationUser = $bDestinationRemote ? "dsteele" : undef;
+            my $strDbUser = $bDbRemote ? "dsteele" : undef;
 
             # Loop through destination compression
             for (my $bDestinationCompressed = 0; $bDestinationCompressed <= 1; $bDestinationCompressed++)
@@ -49,10 +136,10 @@ sub BackRestTestFile
                     bNoCompression => !$bDestinationCompressed,
                     strBackupClusterPath => undef,
                     strBackupPath => ${strTestPath},
-                    strBackupHost => $strSourceHost,
+                    strBackupHost => $strBackupHost,
-                    strBackupUser => $strSourceUser,
+                    strBackupUser => $strBackupUser,
-                    strDbHost => $strDestinationHost,
+                    strDbHost => $strDbHost,
-                    strDbUser => $strDestinationUser,
+                    strDbUser => $strDbUser,
                     strCommandCompress => "gzip --stdout %file%",
                     strCommandDecompress => "gzip -dc %file%",
                     strLockPath => dirname($0) . "/test/lock"
@@ -75,8 +162,8 @@ sub BackRestTestFile
                                     $iRun++;
                                 
                                     print "run ${iRun} - " .
-                                          "srcpth ${strSourcePath}, srcrmt $bSourceRemote, srccmp $bSourceCompressed, " .
+                                          "srcpth ${strSourcePath}, bkprmt $bBackupRemote, srccmp $bSourceCompressed, " .
-                                          "dstpth ${strDestinationPath}, dstrmt $bDestinationRemote, dstcmp $bDestinationCompressed, " .
+                                          "dstpth ${strDestinationPath}, dbrmt $bDbRemote, dstcmp $bDestinationCompressed, " .
                                           "error $bError, confess_error $bConfessError\n";
 
                                     # Drop the old test directory and create a new one
@@ -118,13 +205,22 @@ sub BackRestTestFile
                                     if ($@)
                                     {
                                         # Different remote and destination with different path types should error
-                                        if ($bSourceRemote && $bDestinationRemote)
+                                        if ($bBackupRemote && $bDbRemote && ($strSourcePath ne $strDestinationPath))
                                         {
+                                            print "    different source and remote for same path not supported\n";
                                             next;
                                         }
                                         # If the error was intentional, then also continue
                                         elsif ($bError)
                                         {
+                                            my $strError = $oFile->error_get();
+                                            
+                                            if (!defined($strError) || ($strError eq ''))
+                                            {
+                                                confess 'no error message returned';
+                                            }
+                                            
+                                            print "    error raised: ${strError}\n";
                                             next;
                                         }
                                         # Else this is an unexpected error
@@ -147,6 +243,14 @@ sub BackRestTestFile
                                             }
                                             else
                                             {
+                                                my $strError = $oFile->error_get();
+                                                
+                                                if (!defined($strError) || ($strError eq ''))
+                                                {
+                                                    confess 'no error message returned';
+                                                }
+                                                
+                                                print "    error returned: ${strError}\n";
                                                 next;
                                             }
                                         }
@@ -158,6 +262,8 @@ sub BackRestTestFile
                                         {
                                             confess "error was returned when no error generated";
                                         }
+                                        
+                                        print "    true was returned\n";
                                     }
                                 
                                     # Check for errors after copy