Restore cleans out missing files and some basic unit tests.

bin/pg_backrest.pl

@@ -413,14 +413,44 @@ if (operation_get() eq OP_ARCHIVE_GET)
 }
 
 ####################################################################################################################################
-# OPEN THE LOG FILE
+# Initialize the default file object
 ####################################################################################################################################
-if (defined(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST)))
-{
-    confess &log(ASSERT, 'backup/expire operations must be performed locally on the backup server');
-}
 
-log_file_set(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . "/log/" . param_get(PARAM_STANZA));
+# Open the log file
+log_file_set(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . '/log/' . param_get(PARAM_STANZA));
+
+my $oFile = new BackRest::File
+(
+    param_get(PARAM_STANZA),
+    config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
+    $strRemote,
+    remote_get()
+);
+
+####################################################################################################################################
+# RESTORE
+####################################################################################################################################
+if (operation_get() eq OP_RESTORE)
+{
+    if ($strRemote eq REMOTE_DB)
+    {
+        confess &log(ASSERT, 'restore operation must be performed locally on the db server');
+    }
+
+    # Open the log file
+#    log_file_set(config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH) . '/restore-' . param_get(PARAM_STANZA));
+
+    # Do the restore
+    new BackRest::Restore
+    (
+        config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH),
+        undef,
+        $oFile,
+        param_get(PARAM_FORCE)
+    )->restore;
+
+    remote_exit(0);
+}
 
 ####################################################################################################################################
 # GET MORE CONFIG INFO
@@ -445,15 +475,6 @@ if (!lock_file_create($strLockPath))
     remote_exit(0);
 }
 
-# Initialize the default file object
-my $oFile = new BackRest::File
-(
-    param_get(PARAM_STANZA),
-    config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
-    $strRemote,
-    remote_get()
-);
-
 # Initialize the db object
 my $oDb;
 
@@ -494,19 +515,6 @@ if (operation_get() eq OP_BACKUP)
     operation_set(OP_EXPIRE);
 }
 
-####################################################################################################################################
-# RESTORE
-####################################################################################################################################
-if (operation_get() eq OP_RESTORE)
-{
-    new BackRest::Restore
-    (
-        config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH),
-        undef,
-        $oFile
-    )->restore;
-}
-
 ####################################################################################################################################
 # EXPIRE
 ####################################################################################################################################

lib/BackRest/Config.pm

@@ -19,13 +19,13 @@ use Exporter qw(import);
 our @EXPORT = qw(config_load config_key_load operation_get operation_set param_get
 
                  FILE_MANIFEST FILE_VERSION FILE_POSTMASTER_PID
+                 PATH_LATEST
 
                  OP_ARCHIVE_GET OP_ARCHIVE_PUSH OP_BACKUP OP_RESTORE OP_EXPIRE
 
                  BACKUP_TYPE_FULL BACKUP_TYPE_DIFF BACKUP_TYPE_INCR
 
                  PARAM_CONFIG PARAM_STANZA PARAM_TYPE PARAM_NO_START_STOP PARAM_FORCE PARAM_VERSION PARAM_HELP
-
                  PARAM_TEST PARAM_TEST_DELAY PARAM_TEST_NO_FORK
 
                  CONFIG_SECTION_COMMAND CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP
@@ -50,7 +50,9 @@ use constant
 {
     FILE_MANIFEST       => 'backup.manifest',
     FILE_VERSION        => 'version',
-    FILE_POSTMASTER_PID => 'postmaster.pid'
+    FILE_POSTMASTER_PID => 'postmaster.pid',
+
+    PATH_LATEST         => 'latest'
 };
 
 ####################################################################################################################################

lib/BackRest/Restore.pm

@@ -28,6 +28,7 @@ sub new
     my $strDbClusterPath = shift;   # Database cluster path
     my $strBackupPath = shift;      # Backup to restore
     my $oFile = shift;              # Default file object
+    my $bForce = shift;             # Force the restore even if files are present
 
     # Create the class hash
     my $self = {};
@@ -35,18 +36,19 @@ sub new
 
     # Initialize variables
     $self->{strDbClusterPath} = $strDbClusterPath;
+    $self->{oFile} = $oFile;
+    $self->{bForce} = $bForce;
 
+    # If backup path is not specified then default to latest
     if (defined($strBackupPath))
     {
         $self->{strBackupPath} = $strBackupPath;
     }
     else
     {
-        $self->{strBackupPath} = 'latest';
+        $self->{strBackupPath} = PATH_LATEST;
     }
 
-    $self->{oFile} = $oFile;
-
     return $self;
 }
 
@@ -64,6 +66,8 @@ sub restore
     }
 
     # Make sure the backup path is valid and load the manifest
+    my %oManifest;
+
     if ($self->{oFile}->exists(PATH_BACKUP_CLUSTER, $self->{strBackupPath}))
     {
         # Copy the backup manifest to the db cluster path
@@ -71,7 +75,7 @@ sub restore
                              PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
 
          # Load the manifest into a hash
-         ini_load($self->{oFile}->path_get(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST));
+         ini_load($self->{oFile}->path_get(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST), \%oManifest);
 
          # Remove the manifest now that it is in memory
          $self->{oFile}->remove(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
@@ -80,6 +84,68 @@ sub restore
     {
         confess &log(ERROR, 'backup ' . $self->{strBackupPath} . ' does not exist');
     }
+
+    # Check each restore directory in the manifest and make sure that it exists and is empty.
+    # The --force option can be used to override the empty requirement.
+    foreach my $strPathKey (sort(keys $oManifest{'backup:path'}))
+    {
+        my $strPath = $oManifest{'backup:path'}{$strPathKey};
+
+        if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE,  $strPath))
+        {
+            confess &log(ERROR, "required db path '${strPath}' does not exist");
+        }
+
+        # Load path manifest so it can be compared to delete files/paths/links that are not in the backup
+        my %oPathManifest;
+        $self->{oFile}->manifest(PATH_DB_ABSOLUTE, $strPath, \%oPathManifest);
+
+        foreach my $strName (sort {$b cmp $a} (keys $oPathManifest{name}))
+        {
+            # Skip the root path
+            if ($strName eq '.')
+            {
+                next;
+            }
+
+            # If force was not specified then error if any file is found
+            if (!$self->{bForce})
+            {
+                confess &log(ERROR, "db path '${strPath}' contains files");
+            }
+
+            # Determine the file/path/link type
+            my $strType = 'file';
+
+            if ($oPathManifest{name}{$strName}{type} eq 'd')
+            {
+                $strType = 'path';
+            }
+            elsif ($oPathManifest{name}{$strName}{type} eq 'l')
+            {
+                $strType = 'link';
+            }
+
+            # Check to see if the file/path/link exists in the manifest, if not remove it
+            if (!defined($oManifest{"${strPathKey}:${strType}"}{$strName}))
+            {
+                my $strDelete = "${strPath}/${strName}";
+
+                # If a path then remove it, all the files should have already been deleted since we are going in reverse order
+                if ($strType eq 'path')
+                {
+                    &log(DEBUG, "remove path ${strDelete}");
+                    rmdir($strDelete) or confess &log(ERROR, "unable to delete path ${strDelete}, is it empty?");
+                }
+                # Else delete a file
+                else
+                {
+                    &log(DEBUG, "remove file ${strDelete}");
+                    unlink($strDelete) or confess &log(ERROR, "unable to delete file ${strDelete}");
+                }
+            }
+        }
+    }
 }
 
 1;

test/lib/BackRestTest/BackupTest.pm

@@ -211,11 +211,11 @@ sub BackRestTestBackup_Create
 }
 
 ####################################################################################################################################
-# BackRestTestBackup_ManifestPathCreate
+# BackRestTestBackup_PathCreate
 #
-# Create a path specifying mode and add it to the manifest.
+# Create a path specifying mode.
 ####################################################################################################################################
-sub BackRestTestBackup_ManifestPathCreate
+sub BackRestTestBackup_PathCreate
 {
     my $oManifestRef = shift;
     my $strPath = shift;
@@ -231,6 +231,24 @@ sub BackRestTestBackup_ManifestPathCreate
         BackRestTestCommon_PathCreate($strFinalPath, $strMode);
     }
 
+    return $strFinalPath;
+}
+
+####################################################################################################################################
+# BackRestTestBackup_ManifestPathCreate
+#
+# Create a path specifying mode and add it to the manifest.
+####################################################################################################################################
+sub BackRestTestBackup_ManifestPathCreate
+{
+    my $oManifestRef = shift;
+    my $strPath = shift;
+    my $strSubPath = shift;
+    my $strMode = shift;
+
+    # Create final file location
+    my $strFinalPath = BackRestTestBackup_PathCreate($oManifestRef, $strPath, $strSubPath, $strMode);
+
     # Stat the file
     my $oStat = lstat($strFinalPath);
 
@@ -309,6 +327,30 @@ sub BackRestTestBackup_ManifestTablespaceCreate
     ${$oManifestRef}{"backup:path"}{"tablespace:${iOid}"} = $strPath;
 }
 
+####################################################################################################################################
+# BackRestTestBackup_FileCreate
+#
+# Create a file specifying content, mode, and time.
+####################################################################################################################################
+sub BackRestTestBackup_FileCreate
+{
+    my $oManifestRef = shift;
+    my $strPath = shift;
+    my $strFile = shift;
+    my $strContent = shift;
+    my $lTime = shift;
+    my $strMode = shift;
+
+    # Create actual file location
+    my $strPathFile = ${$oManifestRef}{'backup:path'}{$strPath} . "/${strFile}";
+
+    # Create the file
+    BackRestTestCommon_FileCreate($strPathFile, $strContent, $lTime, $strMode);
+
+    # Return path to created file
+    return $strPathFile;
+}
+
 ####################################################################################################################################
 # BackRestTestBackup_ManifestFileCreate
 #
@@ -324,11 +366,8 @@ sub BackRestTestBackup_ManifestFileCreate
     my $lTime = shift;
     my $strMode = shift;
 
-    # Create actual file location
-    my $strPathFile = ${$oManifestRef}{'backup:path'}{$strPath} . "/${strFile}";
-
     # Create the file
-    BackRestTestCommon_FileCreate($strPathFile, $strContent, $lTime, $strMode);
+    my $strPathFile = BackRestTestBackup_FileCreate($oManifestRef, $strPath, $strFile, $strContent, $lTime, $strMode);
 
     # Stat the file
     my $oStat = lstat($strPathFile);
@@ -541,7 +580,7 @@ sub BackRestTestBackup_CompareRestore
 
     # Create the backup command
     BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_DbPathGet() .
-                               "/pg_backrest.conf --stanza=${strStanza} restore");
+                               "/pg_backrest.conf --force --stanza=${strStanza} restore");
 }
 
 ####################################################################################################################################
@@ -959,6 +998,11 @@ sub BackRestTestBackup_Test
             my $strFullBackup = BackRestTestBackup_LastBackup($oFile);
 
             BackRestTestBackup_CompareBackup($oFile, $bRemote, $strFullBackup, \%oManifest);
+
+            # Create a bogus file that should be deleted by the restore
+            BackRestTestBackup_PathCreate(\%oManifest, 'base', 'deleteme');
+            BackRestTestBackup_FileCreate(\%oManifest, 'base', 'deleteme/deleteme.txt', 'DELETEME');
+
             BackRestTestBackup_CompareRestore($oFile, $strFullBackup, $strStanza, \%oManifest);
 
             # Perform first incr backup