Lots of improvements to unit tests. A few bug fixes.

bin/pg_backrest_remote.pl

@@ -154,7 +154,7 @@ while ($strCommand ne OP_EXIT)
                              PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'),
                              param_get(\%oParamHash, 'source_compressed'),
                              param_get(\%oParamHash, 'destination_compress'),
-                             param_get(\%oParamHash, 'ignore_missing_source', false).
+                             param_get(\%oParamHash, 'ignore_missing_source', false),
                              undef,
                              param_get(\%oParamHash, 'permission', false),
                              param_get(\%oParamHash, 'destination_path_create')) ? 'Y' : 'N');

lib/BackRest/Backup.pm

@@ -1204,7 +1204,8 @@ sub backup_file_thread
 
         # Copy the file from the database to the backup (will return false if the source file is missing)
         unless($oFileThread->copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
-                                  PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file},
+                                  PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file} .
+                                      ($bCompress ? '.' . $oFile->{strCompressExtension} : ''),
                                   false,        # Source is not compressed since it is the db directory
                                   $bCompress,   # Destination should be compressed based on backup settings
                                   true,         # Ignore missing files

lib/BackRest/File.pm

@@ -499,6 +499,7 @@ sub compress
     # Run locally
     else
     {
+        # Compress the file
         if (!gzip($strPathOp => "${strPathOp}.gz"))
         {
             my $strError = "${strPathOp} could not be compressed:" . $!;
@@ -518,6 +519,7 @@ sub compress
             confess &log(ERROR, "${strDebug}: " . $strError);
         }
 
+        # Remove the old file
         unlink($strPathOp)
             or die &log(ERROR, "${strDebug}: unable to remove ${strPathOp}");
     }
@@ -1373,18 +1375,18 @@ sub copy
     if (!$bDestinationRemote)
     {
         # Set the file permission if required
-        # if (defined($strPermission))
+        if (defined($strPermission))
-        # {
+        {
-        #     system("chmod ${strPermission} ${strDestinationTmpOp}") == 0
+            chmod(oct($strPermission), $strDestinationTmpOp)
-        #         or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmpOp}");
+                or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmpOp}");
-        # }
+        }
 
         # Set the file modification time if required
-        # if (defined($lModificationTime))
+        if (defined($lModificationTime))
-        # {
+        {
-        #     utime($lModificationTime, $lModificationTime, $strDestinationTmpOp)
+            utime($lModificationTime, $lModificationTime, $strDestinationTmpOp)
-        #         or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}");
+                or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}");
-        # }
+        }
 
         # Move the file from tmp to final destination
         $self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, true);

test/lib/BackRestTest/BackupTest.pm

@@ -33,9 +33,9 @@ my $strHost;
 my $strUserBackRest;
 
 ####################################################################################################################################
-# BackRestTestBackup_ClusterDrop
+# BackRestTestBackup_ClusterStop
 ####################################################################################################################################
-sub BackRestTestBackup_ClusterDrop
+sub BackRestTestBackup_ClusterStop
 {
     my $strPath = shift;
 
@@ -78,41 +78,66 @@ sub BackRestTestBackup_ClusterCreate
 }
 
 ####################################################################################################################################
-# BackRestTestBackup_Setup
+# BackRestTestBackup_Drop
 ####################################################################################################################################
-sub BackRestTestBackup_Setup
+sub BackRestTestBackup_Drop
 {
-    my $strRemote;
+    # Stop the cluster if one is running
-    my $bDropOnly = shift;
+    BackRestTestBackup_ClusterStop(BackRestTestCommon_DbCommonPathGet());
-
-    BackRestTestBackup_ClusterDrop($strTestPath . "/db/common");
 
     # Remove the backrest private directory
-    if (-e "${strTestPath}/backrest")
+    if (-e BackRestTestCommon_BackupPathGet())
     {
-        BackRestTestCommon_ExecuteBackRest("rm -rf ${strTestPath}/backrest", true);
+        BackRestTestCommon_Execute('rm -rf ' . BackRestTestCommon_BackupPathGet(), true, true);
     }
 
     # Remove the test directory
-    system("rm -rf ${strTestPath}") == 0 or die 'unable to remove ${strTestPath} path';
+    system('rm -rf ' . BackRestTestCommon_TestPathGet()) == 0
+        or die 'unable to remove ' . BackRestTestCommon_TestPathGet() .  'path';
+}
 
-    if (!defined($bDropOnly) || !$bDropOnly)
+####################################################################################################################################
+# BackRestTestBackup_Create
+####################################################################################################################################
+sub BackRestTestBackup_Create
+{
+    my $bRemote = shift;
+
+    # Set defaults
+    $bRemote = defined($bRemote) ? $bRemote : false;
+
+    # Drop the old test directory
+    BackRestTestBackup_Drop();
+
+    # Create the test directory
+    mkdir(BackRestTestCommon_TestPathGet(), oct('0770'))
+        or confess 'Unable to create ' . BackRestTestCommon_TestPathGet() . ' path';
+
+    # Create the db directory
+    mkdir(BackRestTestCommon_DbPathGet(), oct('0700'))
+        or confess 'Unable to create ' . BackRestTestCommon_DbPathGet() . ' path';
+
+    # Create the db/common directory
+    mkdir(BackRestTestCommon_DbCommonPathGet())
+        or confess 'Unable to create ' . BackRestTestCommon_DbCommonPathGet() . ' path';
+
+    # Create the archive directory
+    mkdir(BackRestTestCommon_ArchivePathGet(), oct('0700'))
+        or confess 'Unable to create ' . BackRestTestCommon_ArchivePathGet() . ' path';
+
+    # Create the backup directory
+    if ($bRemote)
     {
-        # Create the test directory
+        BackRestTestCommon_Execute("mkdir -m 700 " . BackRestTestCommon_BackupPathGet(), true);
-        mkdir($strTestPath, oct("0770")) or confess "Unable to create ${strTestPath} path";
-
-        # Create the db directory
-        mkdir($strTestPath . "/db", oct("0700")) or confess "Unable to create ${strTestPath}/db path";
-
-        # Create the db/common directory
-        mkdir($strTestPath . "/db/common") or confess "Unable to create ${strTestPath}/db/common path";
-
-        # Create the cluster
-        BackRestTestBackup_ClusterCreate($strTestPath . "/db/common", BackRestTestCommon_DbPortGet);
-
-        # Create the backrest directory
-        BackRestTestCommon_ExecuteBackRest("mkdir -m 770 ${strTestPath}/backrest")
     }
+    else
+    {
+        mkdir(BackRestTestCommon_BackupPathGet(), oct('0700'))
+            or confess 'Unable to create ' . BackRestTestCommon_BackupPathGet() . ' path';
+    }
+
+    # Create the cluster
+    BackRestTestBackup_ClusterCreate(BackRestTestCommon_DbCommonPathGet(), BackRestTestCommon_DbPortGet());
 }
 
 ####################################################################################################################################
@@ -147,14 +172,12 @@ sub BackRestTestBackup_Test
 
         &log(INFO, "Test Full Backup\n");
 
-        for (my $bRemote = 0; $bRemote <= 1; $bRemote++)
+        for (my $bRemote = false; $bRemote <= true; $bRemote++)
         {
-            BackRestTestBackup_Setup();
+            BackRestTestBackup_Create($bRemote);
 
-            for (my $bHardlink = 0; $bHardlink <= 1; $bHardlink++)
+            for (my $bHardlink = false; $bHardlink <= true; $bHardlink++)
             {
-    #            BackRestTestBackup_ClusterRestart();
-
                 my %oDbConfigHash;
                 my %oBackupConfigHash;
 
@@ -164,33 +187,39 @@ sub BackRestTestBackup_Test
                     $oBackupConfigHash{'global:backup'}{hardlink} = 'y';
                 }
 
-                BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', 'db',
+                # for (my $bArchiveLocal = false; $bArchiveLocal <= true; $bArchiveLocal++)
-                                                ($bRemote ? REMOTE_BACKUP : undef), \%oDbConfigHash);
+                # {
-                BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', 'backup',
+                    BackRestTestCommon_ConfigCreate('db',
-                                                ($bRemote ? REMOTE_DB : undef), \%oBackupConfigHash);
+                                                    ($bRemote ? REMOTE_BACKUP : undef), \%oDbConfigHash);
+                    BackRestTestCommon_ConfigCreate('backup',
+                                                    ($bRemote ? REMOTE_DB : undef), \%oBackupConfigHash);
 
-                for (my $iFull = 1; $iFull <= 1; $iFull++)
+                    for (my $iFull = 1; $iFull <= 1; $iFull++)
-                {
-                    $iRun++;
-
-                    &log(INFO, "run ${iRun} - " .
-                               "remote ${bRemote}, full ${iFull}");
-
-                    BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
-                                               "/pg_backrest.conf --type=full --stanza=${strStanza} backup");
-
-                for (my $iIncr = 1; $iIncr <= 1; $iIncr++)
                     {
                         $iRun++;
 
                         &log(INFO, "run ${iRun} - " .
-                                   "remote ${bRemote}, full ${iFull}, hardlink ${bHardlink}, incr ${iIncr}");
+                                   "remote ${bRemote}, full ${iFull}");
 
-                        BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
+                        my $strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
-                                                   "/pg_backrest.conf --type=incr --stanza=${strStanza} backup");
+                                                   "/pg_backrest.conf --type=incr --stanza=${strStanza} backup";
+
+                        BackRestTestCommon_Execute($strCommand, $bRemote);
+
+                        for (my $iIncr = 1; $iIncr <= 1; $iIncr++)
+                        {
+                            $iRun++;
+
+                            &log(INFO, "run ${iRun} - " .
+                                       "remote ${bRemote}, full ${iFull}, hardlink ${bHardlink}, incr ${iIncr}");
+
+                            BackRestTestCommon_Execute($strCommand, $bRemote);
+                        }
                     }
-                }
+                # }
             }
+
+            BackRestTestBackup_Drop();
         }
     }
 

test/lib/BackRestTest/CommonTest.pm

@@ -26,7 +26,8 @@ our @EXPORT = qw(BackRestTestCommon_Setup BackRestTestCommon_Execute BackRestTes
                  BackRestTestCommon_StanzaGet BackRestTestCommon_CommandMainGet BackRestTestCommon_CommandRemoteGet
                  BackRestTestCommon_HostGet BackRestTestCommon_UserGet BackRestTestCommon_GroupGet
                  BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_BackupPathGet
-                 BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet BackRestTestCommon_DbPortGet);
+                 BackRestTestCommon_ArchivePathGet BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet
+                 BackRestTestCommon_DbPortGet);
 
 my $strCommonStanza;
 my $strCommonCommandMain;
@@ -38,6 +39,7 @@ my $strCommonGroup;
 my $strCommonUserBackRest;
 my $strCommonTestPath;
 my $strCommonBackupPath;
+my $strCommonArchivePath;
 my $strCommonDbPath;
 my $strCommonDbCommonPath;
 my $iCommonDbPort;
@@ -48,30 +50,27 @@ my $iCommonDbPort;
 sub BackRestTestCommon_Execute
 {
     my $strCommand = shift;
+    my $bRemote = shift;
     my $bSuppressError = shift;
 
+    # Set defaults
+    $bRemote = defined($bRemote) ? $bRemote : false;
+    $bSuppressError = defined($bSuppressError) ? $bSuppressError : false;
+
+    if ($bRemote)
+    {
+        $strCommand = "ssh ${strCommonUserBackRest}\@${strCommonHost} '${strCommand}'";
+    }
+
     if (system($strCommand) != 0)
     {
-        if (!defined($bSuppressError) || !$bSuppressError)
+        if (!$bSuppressError)
         {
             confess &log(ERROR, "unable to execute command: ${strCommand}");
         }
     }
 }
 
-####################################################################################################################################
-# BackRestTestBackup_ExecuteBackRest
-####################################################################################################################################
-sub BackRestTestCommon_ExecuteBackRest
-{
-    my $strCommand = shift;
-    my $bSuppressError = shift;
-
-    $strCommand = "ssh ${strCommonUserBackRest}\@${strCommonHost} '${strCommand}'";
-
-    BackRestTestCommon_Execute($strCommand, $bSuppressError);
-}
-
 ####################################################################################################################################
 # BackRestTestCommon_Setup
 ####################################################################################################################################
@@ -87,6 +86,7 @@ sub BackRestTestCommon_Setup
     $strCommonUserBackRest = 'backrest';
     $strCommonTestPath = dirname(abs_path($0)) . '/test';
     $strCommonBackupPath = "${strCommonTestPath}/backrest";
+    $strCommonArchivePath = "${strCommonTestPath}/archive";
     $strCommonDbPath = "${strCommonTestPath}/db";
     $strCommonDbCommonPath = "${strCommonTestPath}/db/common";
     $iCommonDbPort = 6543;
@@ -97,7 +97,6 @@ sub BackRestTestCommon_Setup
 ####################################################################################################################################
 sub BackRestTestCommon_ConfigCreate
 {
-    my $strFile = shift;
     my $strLocal = shift;
     my $strRemote = shift;
     my $oParamHashRef = shift;
@@ -146,9 +145,27 @@ sub BackRestTestCommon_ConfigCreate
         }
     }
 
-    tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}";
+    # Write out the configuration file
+    my $strFile = BackRestTestCommon_TestPathGet() . '/pg_backrest.conf';
 
-    chmod(0770, $strFile) or die "unable to set permissions for ${strFile}";
+    tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}";
+    chmod(0660, $strFile) or die "unable to set permissions for ${strFile}";
+
+    # Move the configuration file based on local
+    if ($strLocal eq 'db')
+    {
+        rename($strFile, BackRestTestCommon_DbPathGet() . '/pg_backrest.conf')
+            or die "unable to move ${strFile} to " . BackRestTestCommon_DbPathGet() . '/pg_backrest.conf path';
+    }
+    elsif ($strLocal eq 'backup' && !defined($strRemote))
+    {
+        rename($strFile, BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf')
+            or die "unable to move ${strFile} to " . BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf path';
+    }
+    else
+    {
+        BackRestTestCommon_Execute("mv $strFile " . BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', true);
+    }
 }
 
 ####################################################################################################################################
@@ -199,6 +216,11 @@ sub BackRestTestCommon_BackupPathGet
     return $strCommonBackupPath;
 }
 
+sub BackRestTestCommon_ArchivePathGet
+{
+    return $strCommonArchivePath;
+}
+
 sub BackRestTestCommon_DbPathGet
 {
     return $strCommonDbPath;