From 5d01aac92e4f27da4f6459885bfd4d00e0d22da6 Mon Sep 17 00:00:00 2001
From: David Steele <github@thelabyrinth.net>
Date: Sun, 22 Jun 2014 16:05:00 -0400
Subject: [PATCH] Working on backup - mostly working except for the last
 archive step.

---
 bin/pg_backrest_remote.pl           | 25 +++++++++++++
 lib/BackRest/Backup.pm              | 55 ++++++++++++-----------------
 lib/BackRest/Db.pm                  |  4 +--
 lib/BackRest/File.pm                | 43 ++++++++++++++--------
 lib/BackRest/Remote.pm              |  2 +-
 test/lib/BackRestTest/BackupTest.pm |  6 ++--
 test/lib/BackRestTest/CommonTest.pm | 16 ++++++++-
 test/lib/BackRestTest/FileTest.pm   |  2 +-
 8 files changed, 98 insertions(+), 55 deletions(-)

diff --git a/bin/pg_backrest_remote.pl b/bin/pg_backrest_remote.pl
index f299749fd..853695bdd 100755
--- a/bin/pg_backrest_remote.pl
+++ b/bin/pg_backrest_remote.pl
@@ -153,6 +153,31 @@ while ($strCommand ne OP_EXIT)
                              param_get(\%oParamHash, 'destination_compress'),
                              param_get(\%oParamHash, 'ignore_missing_source', false)) ? 'Y' : 'N');
         }
+        elsif ($strCommand eq OP_FILE_MANIFEST)
+        {
+            my %oManifestHash;
+
+            $oFile->manifest(PATH_ABSOLUTE, param_get(\%oParamHash, 'path'), \%oManifestHash);
+
+            my $strOutput = "name\ttype\tuser\tgroup\tpermission\tmodification_time\tinode\tsize\tlink_destination";
+
+            foreach my $strName (sort(keys $oManifestHash{name}))
+            {
+                $strOutput .= "\n${strName}\t" .
+                    $oManifestHash{name}{"${strName}"}{type} . "\t" .
+                    (defined($oManifestHash{name}{"${strName}"}{user}) ? $oManifestHash{name}{"${strName}"}{user} : "") . "\t" .
+                    (defined($oManifestHash{name}{"${strName}"}{group}) ? $oManifestHash{name}{"${strName}"}{group} : "") . "\t" .
+                    (defined($oManifestHash{name}{"${strName}"}{permission}) ? $oManifestHash{name}{"${strName}"}{permission} : "") . "\t" .
+                    (defined($oManifestHash{name}{"${strName}"}{modification_time}) ?
+                        $oManifestHash{name}{"${strName}"}{modification_time} : "") . "\t" .
+                    (defined($oManifestHash{name}{"${strName}"}{inode}) ? $oManifestHash{name}{"${strName}"}{inode} : "") . "\t" .
+                    (defined($oManifestHash{name}{"${strName}"}{size}) ? $oManifestHash{name}{"${strName}"}{size} : "") . "\t" .
+                    (defined($oManifestHash{name}{"${strName}"}{link_destination}) ?
+                        $oManifestHash{name}{"${strName}"}{link_destination} : "");
+            }
+
+            $oRemote->output_write($strOutput);
+        }
         else
         {
             if ($strCommand ne OP_NOOP)
diff --git a/lib/BackRest/Backup.pm b/lib/BackRest/Backup.pm
index 3364595f4..8367ea6e8 100644
--- a/lib/BackRest/Backup.pm
+++ b/lib/BackRest/Backup.pm
@@ -29,6 +29,7 @@ my $oDb;
 my $oFile;
 my $strType = "incremental";        # Type of backup: full, differential (diff), incremental (incr)
 my $bHardLink;
+my $bCompress = true;
 my $bNoChecksum;
 my $iThreadMax;
 my $iThreadLocalMax;
@@ -788,8 +789,7 @@ sub backup_manifest_build
         ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{user} = $oManifestHash{name}{"${strName}"}{user};
         ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{group} = $oManifestHash{name}{"${strName}"}{group};
         ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{permission} = $oManifestHash{name}{"${strName}"}{permission};
-        ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} =
-            (split("\\.", $oManifestHash{name}{"${strName}"}{modification_time}))[0];
+        ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} = $oManifestHash{name}{"${strName}"}{modification_time};
 
         if ($cType eq "f")
         {
@@ -1184,23 +1184,13 @@ sub backup_file_thread
                   file_size_format($oFileCopyMap{$strFile}{size}) .
                   ($lSizeTotal > 0 ? ", " . int($lSize * 100 / $lSizeTotal) . "%" : "") . ")";
 
-        # Copy the file from the database to the backup
-        unless($oFileThread->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
-                                              PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file},
-                                              undef, $oFileCopyMap{$strFile}{modification_time},
-                                              undef, $bPathCreate, false))
+        # Copy the file from the database to the backup (will return false if the source file is missing)
+        unless($oFileThread->copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
+                                  PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file},
+                                  false,      # Source is not compressed since it is the db directory
+                                  $bCompress, # Destination should be compressed based on backup settings
+                                  true))      # Ignore missing files
         {
-            &log(DEBUG, "thread ${iThreadIdx} unable to copy file: " . $oFileCopyMap{$strFile}{db_file});
-
-            # If the copy fails then then check if the file exists.  The database frequently removes files so it is normal for
-            # files to be missing after the manifest is built.  However, if the file exists then it means there was some other
-            # sort of fatal copy error and an abort is required to prevent a corrupted backup
-            if ($oFileThread->file_exists(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}))
-            {
-                # !!! Improve this error when able to retrieve error text from the File object
-                confess &log(ERROR, "unable to copy file $oFileCopyMap{$strFile}{db_file}");
-            }
-
             # If file is missing assume the database removed it (else corruption and nothing we can do!)
             &log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file});
 
@@ -1216,20 +1206,20 @@ sub backup_file_thread
         }
 
         # Generate checksum for file if configured
-        if ($bChecksum && $lSize != 0)
-        {
-            # Generate the checksum
-            my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file});
-
-            # Write the checksum message into the master queue
-            $oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
-
-            &log(INFO, $strLog . " checksum ${strChecksum}");
-        }
-        else
-        {
+        # if ($bChecksum && $lSize != 0)
+        # {
+        #     # Generate the checksum
+        #     my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file});
+        #
+        #     # Write the checksum message into the master queue
+        #     $oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
+        #
+        #     &log(INFO, $strLog . " checksum ${strChecksum}");
+        # }
+        # else
+        # {
             &log(INFO, $strLog);
-        }
+        # }
     }
 
     &log(DEBUG, "thread ${iThreadIdx} exiting");
@@ -1259,7 +1249,8 @@ sub backup
     &log(DEBUG, "cluster path is $strDbClusterPath");
 
     # Create the cluster backup path
-    $oFile->path_create(PATH_BACKUP, "backup", undef, true);
+    # $oFile->path_create(PATH_BACKUP, "backup", undef, true);
+    # $oFile->path_create(PATH_BACKUP, "temp", undef, true);
     $oFile->path_create(PATH_BACKUP_CLUSTER, undef, undef, true);
 
     # Find the previous backup based on the type
diff --git a/lib/BackRest/Db.pm b/lib/BackRest/Db.pm
index db009e123..30642fd3c 100644
--- a/lib/BackRest/Db.pm
+++ b/lib/BackRest/Db.pm
@@ -99,8 +99,8 @@ sub tablespace_map_get
     my $self = shift;
     my $oHashRef = shift;
 
-    return data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute(
-                           "copy (select oid, spcname from pg_tablespace) to stdout"), "\t");
+    data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute(
+                    "copy (select oid, spcname from pg_tablespace) to stdout"), "\t");
 }
 
 ####################################################################################################################################
diff --git a/lib/BackRest/File.pm b/lib/BackRest/File.pm
index 833372867..bf89ace1b 100644
--- a/lib/BackRest/File.pm
+++ b/lib/BackRest/File.pm
@@ -13,6 +13,7 @@ use Net::OpenSSH;
 use IPC::Open3;
 use File::Basename;
 use File::Copy qw(cp);
+use File::Path qw(make_path remove_tree);
 use Digest::SHA;
 use File::stat;
 use Fcntl ':mode';
@@ -579,18 +580,18 @@ sub path_create
         if (!($bIgnoreExists && $self->exists($strPathType, $strPath)))
         {
             # Attempt the create the directory
-            my $bResult;
+            my $stryError;
 
             if (defined($strPermission))
             {
-                $bResult = mkdir($strPathOp, oct($strPermission));
+                make_path($strPathOp, {mode => oct($strPermission), error => \$stryError});
             }
             else
             {
-                $bResult = mkdir($strPathOp);
+                make_path($strPathOp, {error => \$stryError});
             }
 
-            if (!$bResult)
+            if (@$stryError)
             {
                 # Capture the error
                 my $strError = "${strPathOp} could not be created: " . $!;
@@ -602,7 +603,7 @@ sub path_create
                 }
 
                 # Error the normal way
-                confess &log(ERROR, "${strDebug}: " . $strError, COMMAND_ERR_PATH_CREATE);
+                confess &log(ERROR, "${strDebug}: " . $strError); #, COMMAND_ERR_PATH_CREATE);
             }
         }
     }
@@ -661,7 +662,7 @@ sub exists
                 }
                 else
                 {
-                    confess &log(ERROR, "${strDebug}: " . $!, COMMAND_ERR_FILE_READ);
+                    confess &log(ERROR, "${strDebug}: " . $!); #, COMMAND_ERR_FILE_READ);
                 }
             }
 
@@ -907,19 +908,30 @@ sub manifest
     my $strPathOp = $self->path_get($strPathType, $strPath);
 
     # Set operation and debug strings
-    my $strOperation = OP_FILE_EXISTS;
+    my $strOperation = OP_FILE_MANIFEST;
     my $strDebug = "${strPathType}:${strPathOp}";
     &log(DEBUG, "${strOperation}: ${strDebug}");
 
     # Run remotely
     if ($self->is_remote($strPathType))
     {
-        confess &log(ASSERT, "${strDebug}: remote operation not supported");
+        # Build param hash
+        my %oParamHash;
+
+        $oParamHash{path} = $strPathOp;
+
+        # Add remote info to debug string
+        my $strRemote = "remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . ")";
+        $strDebug = "${strOperation}: ${strRemote}: ${strDebug}";
+        &log(TRACE, "${strOperation}: ${strRemote}");
+
+        # Execute the command
+        data_hash_build($oManifestHashRef, $self->{oRemote}->command_execute($strOperation, \%oParamHash, true, $strDebug), "\t");
     }
     # Run locally
     else
     {
-        $self->manifest_recurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef);
+        $self->manifest_recurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef, $strDebug);
     }
 }
 
@@ -931,8 +943,9 @@ sub manifest_recurse
     my $strPathFileOp = shift;
     my $iDepth = shift;
     my $oManifestHashRef = shift;
+    my $strDebug = shift;
 
-    my $strErrorPrefix = "File->manifest";
+    $strDebug = $strDebug . (defined($strPathFileOp) ? " => ${strPathFileOp}" : "");
     my $strPathRead = $strPathOp . (defined($strPathFileOp) ? "/${strPathFileOp}" : "");
     my $hPath;
 
@@ -952,7 +965,7 @@ sub manifest_recurse
             confess &log(ERROR, $strError, $iErrorCode);
         }
 
-        confess &log(ERROR, "${strErrorPrefix}: " . $strError);
+        confess &log(ERROR, "${strDebug}: " . $strError);
     }
 
     my @stryFileList = grep(!/^\..$/i, readdir($hPath));
@@ -995,7 +1008,7 @@ sub manifest_recurse
                 confess &log(ERROR, $strError, COMMAND_ERR_FILE_READ);
             }
 
-            confess &log(ERROR, "${strErrorPrefix}: " . $strError, COMMAND_ERR_FILE_READ);
+            confess &log(ERROR, "${strDebug}: " . $strError); #, COMMAND_ERR_FILE_READ);
         }
 
         # Check for regular file
@@ -1037,7 +1050,7 @@ sub manifest_recurse
                         exit COMMAND_ERR_LINK_READ;
                     }
 
-                    confess &log(ERROR, "${strErrorPrefix}: " . $strError);
+                    confess &log(ERROR, "${strDebug}: " . $strError);
                 }
             }
         }
@@ -1051,7 +1064,7 @@ sub manifest_recurse
                 exit COMMAND_ERR_FILE_TYPE;
             }
 
-            confess &log(ERROR, "${strErrorPrefix}: " . $strError);
+            confess &log(ERROR, "${strDebug}: " . $strError);
         }
 
         # Get user name
@@ -1069,7 +1082,7 @@ sub manifest_recurse
         # Recurse into directories
         if (${$oManifestHashRef}{name}{"${strFile}"}{type} eq "d" && !$bCurrentDir)
         {
-            $self->manifest_recurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef);
+            $self->manifest_recurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef, $strDebug);
         }
     }
 }
diff --git a/lib/BackRest/Remote.pm b/lib/BackRest/Remote.pm
index 3d16b8532..2b4d52fee 100644
--- a/lib/BackRest/Remote.pm
+++ b/lib/BackRest/Remote.pm
@@ -24,7 +24,7 @@ use BackRest::Utility;
 ####################################################################################################################################
 use constant
 {
-    DEFAULT_BLOCK_SIZE  => 8192
+    DEFAULT_BLOCK_SIZE  => 1048576
 };
 
 ####################################################################################################################################
diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm
index 21e0d96d9..0f8492242 100755
--- a/test/lib/BackRestTest/BackupTest.pm
+++ b/test/lib/BackRestTest/BackupTest.pm
@@ -124,10 +124,10 @@ sub BackRestTestBackup_Test
 
     BackRestTestBackup_Setup();
 
-    BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);
-    BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_DB);
+    BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_DB);#, REMOTE_BACKUP);
+    BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);#, REMOTE_DB);
     
-    BackRestTestCommon_ExecuteBackRest(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
+    BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
                                        "/pg_backrest.conf --type=incr --stanza=${strStanza} backup");
 
     #-------------------------------------------------------------------------------------------------------------------------------
diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm
index 43ebd03c2..21f1e2934 100755
--- a/test/lib/BackRestTest/CommonTest.pm
+++ b/test/lib/BackRestTest/CommonTest.pm
@@ -98,6 +98,7 @@ sub BackRestTestCommon_Setup
 sub BackRestTestCommon_ConfigCreate
 {
     my $strFile = shift;
+    my $strLocal = shift;
     my $strRemote = shift;
     my $oParamHashRef = shift;
     
@@ -116,12 +117,25 @@ sub BackRestTestCommon_ConfigCreate
     {
         $oParamHash{$strCommonStanza}{'host'} = $strCommonHost;
         $oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
+    }
+    
+    if ($strLocal eq REMOTE_BACKUP)
+    {
         $oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
-
         $oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}";
     }
+    elsif ($strLocal eq REMOTE_DB)
+    {
+    }
+    else
+    {
+        confess "invalid local type ${strLocal}";
+    }
 
     $oParamHash{'global:backup'}{'path'} = $strCommonBackupPath;
+    
+    $oParamHash{'global:log'}{'level-console'} = 'debug';
+    $oParamHash{'global:log'}{'level-file'} = 'trace';
 
     tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}";
     
diff --git a/test/lib/BackRestTest/FileTest.pm b/test/lib/BackRestTest/FileTest.pm
index fc4b86c9d..fd37b7d96 100755
--- a/test/lib/BackRestTest/FileTest.pm
+++ b/test/lib/BackRestTest/FileTest.pm
@@ -498,7 +498,7 @@ sub BackRestTestFile_Test
 
                 # Execute in eval in case of error
                 my %oManifestHash;
-                my $bErrorExpected = !$bExists || $bError || $bRemote;
+                my $bErrorExpected = !$bExists || $bError;
 
                 eval
                 {