Cleanup, work on test framework.

bin/pg_backrest.pl

@@ -6,13 +6,13 @@
 ####################################################################################################################################
 # Perl includes
 ####################################################################################################################################
+use threads;
 use strict;
 use warnings;
-use threads;
+use Carp;
 
 use File::Basename;
 use Getopt::Long;
-use Carp;
 
 use lib dirname($0) . "/../lib";
 use BackRest::Utility;
@@ -75,7 +75,7 @@ my $strType;            # Type of backup: full, differential (diff), incremental
 # Test parameters - not for general use
 my $bNoFork = false;    # Prevents the archive process from forking when local archiving is enabled
 my $bTest = false;      # Enters test mode - not harmful in anyway, but adds special logging and pauses for unit testing
-my $iTestDelay = 5;     # Amount of time to delay after hitting a test point (the default would not be enough for manual tests)
+my $iTestDelay = 2;     # Amount of time to delay after hitting a test point (the default would not be enough for manual tests)
 
 GetOptions ("config=s" => \$strConfigFile,
             "stanza=s" => \$strStanza,
@@ -87,11 +87,8 @@ GetOptions ("config=s" => \$strConfigFile,
             "test-delay=s" => \$iTestDelay)
     or confess("Error in command line arguments\n");
 
-# Test delay should be between 1 and 600 seconds
+# Set test parameters
-if (!($iTestDelay >= 1 && $iTestDelay <= 600))
+test_set($bTest, $iTestDelay);
-{
-    confess &log(ERROR, 'test-delay must be between 1 and 600 seconds');
-}
 
 ####################################################################################################################################
 # Global variables
@@ -99,7 +96,6 @@ if (!($iTestDelay >= 1 && $iTestDelay <= 600))
 my %oConfig;            # Configuration hash
 my $oRemote;            # Remote object
 my $strRemote;          # Defines which side is remote, DB or BACKUP
-my %oConfig;            # Config file loaded into a hash
 
 ####################################################################################################################################
 # CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false)
@@ -213,6 +209,11 @@ $SIG{TERM} = \&safe_exit;
 $SIG{HUP} = \&safe_exit;
 $SIG{INT} = \&safe_exit;
 
+####################################################################################################################################
+# START EVAL BLOCK TO CATCH ERRORS AND STOP THREADS
+####################################################################################################################################
+eval {
+
 ####################################################################################################################################
 # START MAIN
 ####################################################################################################################################
@@ -643,3 +644,14 @@ if ($strOperation eq OP_EXPIRE)
 }
 
 remote_exit(0);
+
+####################################################################################################################################
+# START EVAL BLOCK TO CATCH ERRORS AND STOP THREADS
+####################################################################################################################################
+};
+
+if ($@)
+{
+    remote_exit();
+    confess $@;
+}

lib/BackRest/Backup.pm

@@ -4,10 +4,10 @@
 package BackRest::Backup;
 
 use threads;
-
 use strict;
 use warnings;
 use Carp;
+
 use File::Basename;
 use File::Path qw(remove_tree);
 use Scalar::Util qw(looks_like_number);
@@ -35,8 +35,6 @@ my $iThreadThreshold = 10;
 my $iSmallFileThreshold = 65536;
 my $bArchiveRequired;
 my $iThreadTimeout;
-my $bTest;
-my $iTestDelay;
 
 # Thread variables
 my @oThread;
@@ -58,8 +56,6 @@ sub backup_init
     my $iThreadMaxParam = shift;
     my $bArchiveRequiredParam = shift;
     my $iThreadTimeoutParam = shift;
-    my $bTestParam = shift;
-    my $iTestDelayParam = shift;
 
     $oDb = $oDbParam;
     $oFile = $oFileParam;
@@ -70,14 +66,6 @@ sub backup_init
     $iThreadMax = $iThreadMaxParam;
     $bArchiveRequired = $bArchiveRequiredParam;
     $iThreadTimeout = $iThreadTimeoutParam;
-    $bTest = defined($bTestParam) ? $bTestParam : false;
-    $iTestDelay = defined($bTestParam) ? $iTestDelayParam : undef;
-
-    # Make sure that a delay is specified in test mode
-    if ($bTest && !defined($iTestDelay))
-    {
-        confess &log(ASSERT, "iTestDelay must be provided when bTest is true");
-    }
 
     if (!defined($iThreadMax))
     {
@@ -1364,6 +1352,7 @@ sub backup
     $oDb->tablespace_map_get(\%oTablespaceMap);
 
     backup_manifest_build($oFile->{strCommandManifest}, $strDbClusterPath, \%oBackupManifest, \%oLastManifest, \%oTablespaceMap);
+    &log(TEST, TEST_MANIFEST_BUILD);
 
     # If the backup tmp path already exists, remove invalid files
     if (-e $strBackupTmpPath)

lib/BackRest/Db.pm

@@ -4,11 +4,11 @@
 package BackRest::Db;
 
 use threads;
-
-use Moose;
 use strict;
 use warnings;
 use Carp;
+
+use Moose;
 use Net::OpenSSH;
 use File::Basename;
 use IPC::System::Simple qw(capture);

lib/BackRest/Utility.pm

@@ -4,10 +4,10 @@
 package BackRest::Utility;
 
 use threads;
-
 use strict;
 use warnings;
 use Carp;
+
 use Fcntl qw(:DEFAULT :flock);
 use File::Path qw(remove_tree);
 use File::Basename;
@@ -20,10 +20,11 @@ use Exporter qw(import);
 
 our @EXPORT = qw(version_get
                  data_hash_build trim common_prefix wait_for_file date_string_get file_size_format execute
-                 log log_file_set log_level_set
+                 log log_file_set log_level_set test_set
                  lock_file_create lock_file_remove
                  config_save config_load
-                 TRACE DEBUG ERROR ASSERT WARN INFO OFF true false);
+                 TRACE DEBUG ERROR ASSERT WARN INFO OFF true false
+                 TEST TEST_ENCLOSE TEST_MANIFEST_BUILD);
 
 # Global constants
 use constant
@@ -59,6 +60,20 @@ $oLogLevelRank{ERROR}{rank} = 2;
 $oLogLevelRank{ASSERT}{rank} = 1;
 $oLogLevelRank{OFF}{rank} = 0;
 
+####################################################################################################################################
+# TEST Constants and Variables
+####################################################################################################################################
+use constant
+{
+    TEST                => 'TEST',
+    TEST_ENCLOSE        => 'PgBaCkReStTeSt',
+    TEST_MANIFEST_BUILD => 'MANIFEST_BUILD'
+};
+
+# Test global variables
+my $bTest = false;
+my $iTestDelay = 2;
+
 ####################################################################################################################################
 # VERSION_GET
 ####################################################################################################################################
@@ -321,6 +336,30 @@ sub log_file_set
     print $hLogFile "-------------------PROCESS START-------------------\n";
 }
 
+####################################################################################################################################
+# TEST_SET - set test parameters
+####################################################################################################################################
+sub test_set
+{
+    my $bTestParam = shift;
+    my $iTestDelayParam = shift;
+
+    $bTest = defined($bTestParam) ? $bTestParam : false;
+    $iTestDelay = defined($bTestParam) ? $iTestDelayParam : $iTestDelay;
+
+    # Test delay should be between 1 and 600 seconds
+    if (!($iTestDelay >= 1 && $iTestDelay <= 600))
+    {
+        confess &log(ERROR, 'test-delay must be between 1 and 600 seconds');
+    }
+
+    # Make sure that a delay is specified in test mode
+    if ($bTest && !defined($iTestDelay))
+    {
+        confess &log(ASSERT, "iTestDelay must be provided when bTest is true");
+    }
+}
+
 ####################################################################################################################################
 # LOG_LEVEL_SET - set the log level for file and console
 ####################################################################################################################################
@@ -361,7 +400,11 @@ sub log
 
     my $strMessageFormat = $strMessage;
 
-    if (!defined($oLogLevelRank{"${strLevel}"}{rank}))
+    if ($bTest && $strLevel eq TEST)
+    {
+        $strMessageFormat = TEST_ENCLOSE . '-' . $strMessageFormat . '-' . TEST_ENCLOSE;
+    }
+    elsif (!defined($oLogLevelRank{"${strLevel}"}{rank}))
     {
         confess &log(ASSERT, "log level ${strLevel} does not exist");
     }
@@ -393,7 +436,8 @@ sub log
                         (" " x (7 - length($strLevel))) . "${strLevel}: ${strMessageFormat}" .
                         (defined($iCode) ? " (code ${iCode})" : "") . "\n";
 
-    if ($oLogLevelRank{"${strLevel}"}{rank} <= $oLogLevelRank{"${strLogLevelConsole}"}{rank})
+    if ($oLogLevelRank{"${strLevel}"}{rank} <= $oLogLevelRank{"${strLogLevelConsole}"}{rank} ||
+        $bTest && $strLevel eq TEST)
     {
         print $strMessageFormat;
     }
@@ -411,6 +455,11 @@ sub log
         return BackRest::Exception->new(iCode => $iCode, strMessage => $strMessage);
     }
 
+    if ($bTest && $strLevel eq TEST)
+    {
+        sleep($iTestDelay);
+    }
+
     return $strMessage;
 }
 

test/lib/BackRestTest/BackupTest.pm

@@ -499,7 +499,7 @@ sub BackRestTestBackup_Test
 
                     my $strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' .
                                                ($bRemote ? BackRestTestCommon_BackupPathGet() : BackRestTestCommon_DbPathGet()) .
-                                               "/pg_backrest.conf --type=incr --stanza=${strStanza} backup";
+                                               "/pg_backrest.conf --test --type=incr --stanza=${strStanza} backup";
 
                     BackRestTestCommon_Execute($strCommand, $bRemote);
                     # exit 0;

test/lib/BackRestTest/CommonTest.pm

@@ -99,80 +99,53 @@ sub BackRestTestCommon_Execute
         $strCommand = "ssh ${strCommonUserBackRest}\@${strCommonHost} '${strCommand}'";
     }
 
-#    system($strCommand);
+    # Create error and out file handles and buffers
-#    my $strError = '';
     my $strErrorLog = '';
     my $hError;
-#    open($hError, '>', BackRestTestCommon_TestPathGet() . '/stderr.log');# or confess "unable to open handle to stderr string: $!\n";
-#    open($hError, '>', \$strError) or confess "unable to open handle to stderr string: $!\n";
 
-#    my $strOut = '';
     my $strOutLog = '';
     my $hOut;
-#    open($hOut, '>', BackRestTestCommon_TestPathGet() . '/stdout.log');# or confess "unable to open handle to stderr string: $!\n";
-#    open($hOut, '>', \$strOut) or confess "unable to open handle to stdout string: $!\n";
 
+    # Execute the command
     my $pId = open3(undef, $hOut, $hError, $strCommand);
+
+    # Create select objects
     my $oErrorSelect = IO::Select->new();
     $oErrorSelect->add($hError);
     my $oOutSelect = IO::Select->new();
     $oOutSelect->add($hOut);
 
-    # Wait for the process to finish and report any errors
+    # While the process is running drain the stdout and stderr streams
-#    my $iExitStatus;
-
-#    waitpid($pId, 0);
-
     while(waitpid($pId, WNOHANG) == 0)
     {
-#        print "stuck here\n";
+        # Drain the stderr stream
-
         if ($oErrorSelect->can_read(.1))
         {
-#            print "read err\n";
-
             while (my $strLine = readline($hError))
             {
-#                print "out: ${strLine}";
                 $strErrorLog .= $strLine;
             }
         }
 
+        # Drain the stdout stream
         if ($oOutSelect->can_read(.1))
         {
-#            print "read out begin\n";
-
             while (my $strLine = readline($hOut))
             {
-#                print "out: ${strLine}";
                 $strOutLog .= $strLine;
             }
-
-#            print "read out end\n";
         }
     }
 
-#    print "got out\n";
+    # Check the exit status and output an error if needed
-
     my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
-#    $iExitStatus = $iExitStatus >> 8;
 
     if ($iExitStatus != 0 && !$bSuppressError)
     {
-        print "${strErrorLog}";
+        confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus .
-
+                     ($strOutLog ne '' ? "STDOUT:\n${strOutLog}" : '') .
-        confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus);
+                     ($strErrorLog ne '' ? "STDERR:\n${strErrorLog}" : ''));
     }
-
-#    print "${strOutLog}\n";
-
-#    close($hError);
-#    close($hOut);
-
-    # while (my $strLine = readline($hOut))
-    # {
-    #     print $strLine;
-    # }
 }
 
 ####################################################################################################################################
@@ -220,10 +193,8 @@ sub BackRestTestCommon_ConfigCreate
     my $iThreadMax = shift;
     my $bArchiveLocal = shift;
     my $bCompressAsync = shift;
-#    my $oParamHashRef = shift;
 
     my %oParamHash;
-#    tie %oParamHash, 'Config::IniFiles';
 
     if (defined($strRemote))
     {
@@ -243,7 +214,7 @@ sub BackRestTestCommon_ConfigCreate
         $oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
     }
 
-    $oParamHash{'global:log'}{'level-console'} = 'trace';
+    $oParamHash{'global:log'}{'level-console'} = 'error';
     $oParamHash{'global:log'}{'level-file'} = 'trace';
 
     if ($strLocal eq REMOTE_BACKUP)
@@ -298,14 +269,6 @@ sub BackRestTestCommon_ConfigCreate
         $oParamHash{'global:backup'}{'thread-max'} = $iThreadMax;
     }
 
-    # foreach my $strSection (keys $oParamHashRef)
-    # {
-    #     foreach my $strKey (keys ${$oParamHashRef}{$strSection})
-    #     {
-    #         $oParamHash{$strSection}{$strKey} = ${$oParamHashRef}{$strSection}{$strKey};
-    #     }
-    # }
-
     # Write out the configuration file
     my $strFile = BackRestTestCommon_TestPathGet() . '/pg_backrest.conf';
     config_save($strFile, \%oParamHash);

test/lib/BackRestTest/UtilityTest.pm

@@ -78,17 +78,6 @@ sub BackRestTestUtility_Test
             # Create the test directory
             if ($bCreate)
             {
-#                $oFile = BackRest::File->new();
-
-                # # Create the file object
-                # $oFile = (BackRest::File->new
-                # (
-                #     # strStanza => $strStanza,
-                #     # strBackupPath => BackRestTestCommon_BackupPathGet(),
-                #     # strRemote => $bRemote ? 'backup' : undef,
-                #     # oRemote => $bRemote ? $oRemote : undef
-                # ))->clone();
-
                 BackRestTestUtility_Create();
 
                 $bCreate = false;