Working on backup unit tests.

bin/pg_backrest.pl

@@ -235,7 +235,7 @@ my $oRemote;
 
 if ($strRemote ne REMOTE_NONE)
 {
-    my $oRemote = BackRest::Remote->new
+    $oRemote = BackRest::Remote->new
     (
         strHost => config_load($strRemote eq REMOTE_DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST, true),
         strUser => config_load($strRemote eq REMOTE_DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_USER, true),
@@ -261,7 +261,7 @@ if ($strOperation eq OP_ARCHIVE_GET)
     }
 
     # Init the file object
-    my $oFile = pg_backrest_file->new
+    my $oFile = BackRest::File->new
     (
         strStanza => $strStanza,
         strRemote => $strRemote,
@@ -337,7 +337,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
         my $bChecksum = config_load($strSection, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
 
         # Run file_init_archive - this is the minimal config needed to run archiving
-        my $oFile = pg_backrest_file->new
+        my $oFile = BackRest::File->new
         (
             # strStanza => $strStanza,
             # bNoCompression => !$bCompress,
@@ -408,7 +408,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
         eval
         {
             # Run file_init_archive - this is the minimal config needed to run archive pulling
-            my $oFile = pg_backrest_file->new
+            my $oFile = BackRest::File->new
             (
                 # strStanza => $strStanza,
                 # bNoCompression => !$bCompress,
@@ -449,7 +449,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
                 &log(WARN, "errors during transfer, starting compression");
 
                 # Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file
-                my $oFile = pg_backrest_file->new
+                my $oFile = BackRest::File->new
                 (
                     # strStanza => $strStanza,
                     # bNoCompression => false,
@@ -499,7 +499,7 @@ log_file_set(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . "/log/$
 # GET MORE CONFIG INFO
 ####################################################################################################################################
 # Make sure backup and expire operations happen on the db side
-if ($strRemote eq backup)
+if ($strRemote eq REMOTE_BACKUP)
 {
     confess &log(ERROR, 'backup and expire operations must run on the backup host');
 }
@@ -536,7 +536,7 @@ if (!lock_file_create($strLockPath))
 }
 
 # Run file_init_archive - the rest of the file config required for backup and restore
-my $oFile = pg_backrest_file->new
+my $oFile = BackRest::File->new
 (
     strStanza => $strStanza,
     strRemote => $strRemote,
@@ -544,7 +544,7 @@ my $oFile = pg_backrest_file->new
     strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true)
 );
 
-my $oDb = pg_backrest_db->new
+my $oDb = BackRest::Db->new
 (
     strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
     strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),

lib/BackRest/Backup.pm

@@ -224,7 +224,7 @@ sub archive_get
     my $strArchivePath = dirname($oFile->path_get(PATH_BACKUP_ARCHIVE, $strSourceArchive));
 
     # Get the name of the requested archive file (may have hash info and compression extension)
-    my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath,
+    my @stryArchiveFile = $oFile->list(PATH_BACKUP_ABSOLUTE, $strArchivePath,
         "^${strSourceArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$");
 
     # If there is more than one matching archive file then there is a serious issue - likely a bug in the archiver
@@ -372,7 +372,7 @@ sub archive_pull
 
     &log(DEBUG, "local archive path max = ${strPathMax}");
 
-    foreach my $strPath ($oFile->file_list_get(PATH_DB_ABSOLUTE, $strArchivePath, "^[0-F]{16}\$"))
+    foreach my $strPath ($oFile->list(PATH_DB_ABSOLUTE, $strArchivePath, "^[0-F]{16}\$"))
     {
         if ($strPath lt $strPathMax)
         {
@@ -572,12 +572,12 @@ sub backup_type_find
 
     if ($strType eq 'incremental')
     {
-        $strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"))[0];
+        $strDirectory = ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"))[0];
     }
 
     if (!defined($strDirectory) && $strType ne "full")
     {
-        $strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"))[0];
+        $strDirectory = ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"))[0];
     }
 
     return $strDirectory;
@@ -755,10 +755,11 @@ sub backup_manifest_build
         $strLevel = "base";
     }
 
-    my %oManifestHash = $oFile->manifest_get(PATH_DB_ABSOLUTE, $strDbClusterPath);
+    my %oManifestHash;
-    my $strName;
 
-    foreach $strName (sort(keys $oManifestHash{name}))
+    $oFile->manifest(PATH_DB_ABSOLUTE, $strDbClusterPath, \%oManifestHash);
+
+    foreach my $strName (sort(keys $oManifestHash{name}))
     {
         # Skip certain files during backup
         if ($strName =~ /^pg\_xlog\/.*/ ||    # pg_xlog/ - this will be reconstructed
@@ -1258,7 +1259,8 @@ sub backup
     &log(DEBUG, "cluster path is $strDbClusterPath");
 
     # Create the cluster backup path
-    $oFile->path_create(PATH_BACKUP_CLUSTER);
+    $oFile->path_create(PATH_BACKUP, "backup", undef, true);
+    $oFile->path_create(PATH_BACKUP_CLUSTER, undef, undef, true);
 
     # Find the previous backup based on the type
     my $strBackupLastPath = backup_type_find($strType, $oFile->path_get(PATH_BACKUP_CLUSTER));
@@ -1317,7 +1319,8 @@ sub backup
     &log(INFO, 'archive start: ' . ${oBackupManifest}{backup}{"archive-start"});
 
     # Build the backup manifest
-    my %oTablespaceMap = $oDb->tablespace_map_get();
+    my %oTablespaceMap;
+    $oDb->tablespace_map_get(\%oTablespaceMap);
 
     backup_manifest_build($oFile->{strCommandManifest}, $strDbClusterPath, \%oBackupManifest, \%oLastManifest, \%oTablespaceMap);
 
@@ -1369,7 +1372,7 @@ sub backup
 
             wait_for_file($strArchivePath, "^${strArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$", 600);
 
-            my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath,
+            my @stryArchiveFile = $oFile->list(PATH_BACKUP_ABSOLUTE, $strArchivePath,
                 "^${strArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$");
 
             if (scalar @stryArchiveFile != 1)
@@ -1487,13 +1490,13 @@ sub backup_expire
         }
 
         my $iIndex = $iFullRetention;
-        @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
+        @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
 
         while (defined($stryPath[$iIndex]))
         {
             # Delete all backups that depend on the full backup.  Done in reverse order so that remaining backups will still
             # be consistent if the process dies
-            foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse"))
+            foreach $strPath ($oFile->list(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse"))
             {
                 system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}");
             }
@@ -1513,12 +1516,12 @@ sub backup_expire
             confess &log(ERROR, "differential_rentention must be a number >= 1");
         }
 
-        @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse");
+        @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse");
 
         if (defined($stryPath[$iDifferentialRetention]))
         {
             # Get a list of all differential and incremental backups
-            foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse"))
+            foreach $strPath ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse"))
             {
                 # Remove all differential and incremental backups before the oldest valid differential
                 if (substr($strPath, 0, length($strPath) - 1) lt $stryPath[$iDifferentialRetention])
@@ -1540,15 +1543,15 @@ sub backup_expire
     # Determine which backup type to use for archive retention (full, differential, incremental)
     if ($strArchiveRetentionType eq "full")
     {
-        @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
+        @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
     }
     elsif ($strArchiveRetentionType eq "differential" || $strArchiveRetentionType eq "diff")
     {
-        @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 0), "reverse");
+        @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 0), "reverse");
     }
     elsif ($strArchiveRetentionType eq "incremental" || $strArchiveRetentionType eq "incr")
     {
-        @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse");
+        @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse");
     }
     else
     {
@@ -1607,7 +1610,7 @@ sub backup_expire
     &log(INFO, "archive retention starts at " . $strArchiveLast);
 
     # Remove any archive directories or files that are out of date
-    foreach $strPath ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, undef, "^[0-F]{16}\$"))
+    foreach $strPath ($oFile->list(PATH_BACKUP_ARCHIVE, undef, "^[0-F]{16}\$"))
     {
         &log(DEBUG, "found major archive path " . $strPath);
 
@@ -1626,7 +1629,7 @@ sub backup_expire
             my $strSubPath;
 
             # Look for archive files in the archive directory
-            foreach $strSubPath ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, $strPath, "^[0-F]{24}.*\$"))
+            foreach $strSubPath ($oFile->list(PATH_BACKUP_ARCHIVE, $strPath, "^[0-F]{24}.*\$"))
             {
                 # Delete if the first 24 characters less than the current archive file
                 if ($strSubPath lt substr($strArchiveLast, 0, 24))

lib/BackRest/Db.pm

@@ -97,8 +97,9 @@ sub psql_execute
 sub tablespace_map_get
 {
     my $self = shift;
+    my $oHashRef = shift;
 
-    return data_hash_build("oid\tname\n" . $self->psql_execute(
+    return data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute(
                            "copy (select oid, spcname from pg_tablespace) to stdout"), "\t");
 }
 

lib/BackRest/File.pm

@@ -134,7 +134,7 @@ sub BUILD
     my $self = shift;
 
     # If remote is defined check parameters and open session
-    if (defined($self->{strRemote}))
+    if (defined($self->{strRemote}) && $self->{strRemote} ne REMOTE_NONE)
     {
         # Make sure remote is valid
         if ($self->{strRemote} ne REMOTE_DB && $self->{strRemote} ne REMOTE_BACKUP)
@@ -544,6 +544,7 @@ sub path_create
     my $strPathType = shift;
     my $strPath = shift;
     my $strPermission = shift;
+    my $bIgnoreExists = shift;
 
     # Set operation variables
     my $strPathOp = $self->path_get($strPathType, $strPath);
@@ -574,6 +575,8 @@ sub path_create
         $self->{oRemote}->command_execute($strOperation, \%oParamHash, false, $strDebug);
     }
     else
+    {
+        if (!($bIgnoreExists && $self->exists($strPathType, $strPath)))
         {
             # Attempt the create the directory
             my $bResult;
@@ -602,6 +605,7 @@ sub path_create
                 confess &log(ERROR, "${strDebug}: " . $strError, COMMAND_ERR_PATH_CREATE);
             }
         }
+    }
 }
 
 ####################################################################################################################################

lib/BackRest/Utility.pm

@@ -294,6 +294,11 @@ sub log_file_set
 {
     my $strFile = shift;
     
+    unless (-e dirname($strFile))
+    {
+        mkdir(dirname($strFile)) or die "unable to create directory for log file ${strFile}";
+    }
+
     $strFile .= "-" . date_string_get("%4d%02d%02d") . ".log";
     my $bExists = false;
 

test/lib/BackRestTest/BackupTest.pm

@@ -55,7 +55,7 @@ sub BackRestTestBackup_ClusterCreate
     my $iPort = shift;
 
     BackRestTestCommon_Execute("initdb -D $strPath -A trust");
-    BackRestTestCommon_Execute("pg_ctl start -o \"-c port=$iPort\" -D $strPath -l $strPath/postgresql.log -w -s");
+    BackRestTestCommon_Execute("pg_ctl start -o \"-c port=$iPort -c checkpoint_segments=1 -c wal_level=archive -c archive_mode=on -c archive_command=true\" -D $strPath -l $strPath/postgresql.log -w -s");
 }
 
 ####################################################################################################################################
@@ -63,6 +63,7 @@ sub BackRestTestBackup_ClusterCreate
 ####################################################################################################################################
 sub BackRestTestBackup_Setup
 {
+    my $strRemote;
     my $bDropOnly = shift;
 
     BackRestTestBackup_ClusterDrop($strTestPath . "/db/common");
@@ -91,7 +92,7 @@ sub BackRestTestBackup_Setup
         BackRestTestBackup_ClusterCreate($strTestPath . "/db/common", BackRestTestCommon_DbPortGet);
 
         # Create the backrest directory
-        BackRestTestCommon_ExecuteBackRest("mkdir -m 700 ${strTestPath}/backrest")
+        BackRestTestCommon_ExecuteBackRest("mkdir -m 770 ${strTestPath}/backrest")
     }
 }
 
@@ -123,15 +124,11 @@ sub BackRestTestBackup_Test
 
     BackRestTestBackup_Setup();
 
-    #-------------------------------------------------------------------------------------------------------------------------------
+    BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);
-    # Create remote
+    BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_DB);
-    #-------------------------------------------------------------------------------------------------------------------------------
+    
-    # my $oRemote = BackRest::Remote->new
+    BackRestTestCommon_ExecuteBackRest(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
-    # (
+                                       "/pg_backrest.conf --type=incr --stanza=${strStanza} backup");
-    #     strHost => $strHost,
-    #     strUser => $strUser,
-    #     strCommand => BackRestTestCommon_CommandRemoteGet(),
-    # );
 
     #-------------------------------------------------------------------------------------------------------------------------------
     # Test path_create()
@@ -255,7 +252,7 @@ sub BackRestTestBackup_Test
     #     }
     # }
 
-    BackRestTestBackup_Setup(true);
+#    BackRestTestBackup_Setup(true);
 }
 
 1;

test/lib/BackRestTest/CommonTest.pm

@@ -14,23 +14,32 @@ use Carp;
 
 use File::Basename;
 use Cwd 'abs_path';
+use Config::IniFiles;
 
 use lib dirname($0) . "/../lib";
 use BackRest::Utility;
+use BackRest::File;
 
 use Exporter qw(import);
 our @EXPORT = qw(BackRestTestCommon_Setup BackRestTestCommon_Execute BackRestTestCommon_ExecuteBackRest
-                 BackRestTestCommon_StanzaGet BackRestTestCommon_CommandRemoteGet
+                 BackRestTestCommon_ConfigCreate
+                 BackRestTestCommon_StanzaGet BackRestTestCommon_CommandMainGet BackRestTestCommon_CommandRemoteGet
                  BackRestTestCommon_HostGet BackRestTestCommon_UserGet BackRestTestCommon_GroupGet
-                 BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_DbPortGet);
+                 BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_BackupPathGet
+                 BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet BackRestTestCommon_DbPortGet);
 
 my $strCommonStanza;
+my $strCommonCommandMain;
 my $strCommonCommandRemote;
+my $strCommonCommandPsql;
 my $strCommonHost;
 my $strCommonUser;
 my $strCommonGroup;
 my $strCommonUserBackRest;
 my $strCommonTestPath;
+my $strCommonBackupPath;
+my $strCommonDbPath;
+my $strCommonDbCommonPath;
 my $iCommonDbPort;
 
 ####################################################################################################################################
@@ -69,15 +78,56 @@ sub BackRestTestCommon_ExecuteBackRest
 sub BackRestTestCommon_Setup
 {
     $strCommonStanza = "db";
-    $strCommonCommandRemote = "/Users/dsteele/pg_backrest/bin/pg_backrest_remote.pl";
+    $strCommonCommandMain = '/Users/dsteele/pg_backrest/bin/pg_backrest.pl';
-    $strCommonHost = "127.0.0.1";
+    $strCommonCommandRemote = '/Users/dsteele/pg_backrest/bin/pg_backrest_remote.pl';
+    $strCommonCommandPsql = '/Library/PostgreSQL/9.3/bin/psql -X %option%';
+    $strCommonHost = '127.0.0.1';
     $strCommonUser = getpwuid($<);
     $strCommonGroup = getgrgid($();
     $strCommonUserBackRest = 'backrest';
-    $strCommonTestPath = dirname(abs_path($0)) . "/test";
+    $strCommonTestPath = dirname(abs_path($0)) . '/test';
+    $strCommonBackupPath = "${strCommonTestPath}/backrest";
+    $strCommonDbPath = "${strCommonTestPath}/db";
+    $strCommonDbCommonPath = "${strCommonTestPath}/db/common";
     $iCommonDbPort = 6543;
 }
 
+####################################################################################################################################
+# BackRestTestCommon_ConfigCreate
+####################################################################################################################################
+sub BackRestTestCommon_ConfigCreate
+{
+    my $strFile = shift;
+    my $strRemote = shift;
+    my $oParamHashRef = shift;
+    
+    my %oParamHash;
+    tie %oParamHash, 'Config::IniFiles';
+
+    $oParamHash{'global:command'}{'remote'} = $strCommonCommandRemote;
+    $oParamHash{'global:command'}{'psql'} = $strCommonCommandPsql;
+
+    if (defined($strRemote) && $strRemote eq REMOTE_BACKUP)
+    {
+        $oParamHash{'global:backup'}{'host'} = $strCommonHost;
+        $oParamHash{'global:backup'}{'user'} = $strCommonUserBackRest;
+    }
+    elsif (defined($strRemote) && $strRemote eq REMOTE_DB)
+    {
+        $oParamHash{$strCommonStanza}{'host'} = $strCommonHost;
+        $oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
+        $oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
+
+        $oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}";
+    }
+
+    $oParamHash{'global:backup'}{'path'} = $strCommonBackupPath;
+
+    tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}";
+    
+    chmod(0770, $strFile) or die "unable to set permissions for ${strFile}";
+}
+
 ####################################################################################################################################
 # Get Methods
 ####################################################################################################################################
@@ -86,6 +136,11 @@ sub BackRestTestCommon_StanzaGet
     return $strCommonStanza;
 }
 
+sub BackRestTestCommon_CommandMainGet
+{
+    return $strCommonCommandMain;
+}
+
 sub BackRestTestCommon_CommandRemoteGet
 {
     return $strCommonCommandRemote;
@@ -116,6 +171,21 @@ sub BackRestTestCommon_TestPathGet
     return $strCommonTestPath;
 }
 
+sub BackRestTestCommon_BackupPathGet
+{
+    return $strCommonBackupPath;
+}
+
+sub BackRestTestCommon_DbPathGet
+{
+    return $strCommonDbPath;
+}
+
+sub BackRestTestCommon_DbCommonPathGet
+{
+    return $strCommonDbCommonPath;
+}
+
 sub BackRestTestCommon_DbPortGet
 {
     return $iCommonDbPort;