v0.15: Added archive-get

* Added archive-get functionality to aid in restores. * Added option to force a checkpoint when starting the backup (start_fast=y).
2025-09-16 09:06:18 +02:00 · 2014-03-29 18:16:08 -04:00
parent 9d949b7ad6
commit dafaeb0b82
7 changed files with 167 additions and 52 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 **/*~
 *~
+
+*.swp
--- a/README.md
+++ b/README.md
@@ -3,9 +3,24 @@ pg_backrest

 Simple Postgres Backup and Restore

+planned for next release
+========================
+
+* Default restore.conf is written to each backup.
+
+* Able to set timeout on ssh connection in config file.
+
 release notes
 =============

+v0.15: Added archive-get
+
+* Added archive-get functionality to aid in restores.
+
+* Added option to force a checkpoint when starting the backup (start_fast=y).
+
+-------------
+
 v0.11: Minor fixes

 Tweaking a few settings after running backups for about a month.
@@ -14,6 +29,8 @@ Tweaking a few settings after running backups for about a month.

 * Changed lock file conflicts on backup and expire commands to ERROR.  They were set to DEBUG due to a copy-and-paste from the archive locks.

+-------------
+
 v0.10: Backup and archiving are functional

 This version has been put into production at Resonate, so it does work, but there are a number of major caveats.
--- a/pg_backrest.conf
+++ b/pg_backrest.conf
@@ -18,6 +18,7 @@ path=/Users/backrest/test
 archive-required=y
 thread-max=2
 thread-timeout=900
+start_fast=y

 [global:archive]
 path=/Users/dsteele/test
--- a/pg_backrest.pl
+++ b/pg_backrest.pl
@@ -1,9 +1,14 @@
 #!/usr/bin/perl
+####################################################################################################################################
+# pg_backrest.pl - Simple Postgres Backup and Restore
+####################################################################################################################################

-use threads;
-
+####################################################################################################################################
+# Perl includes
+####################################################################################################################################
 use strict;
 use warnings;
+use threads;

 use File::Basename;
 use Getopt::Long;
@@ -16,15 +21,21 @@ use pg_backrest_file;
 use pg_backrest_backup;
 use pg_backrest_db;

-# Operation constants
+####################################################################################################################################
+# Operation constants - basic operations that are allowed in backrest
+####################################################################################################################################
 use constant
 {
+    OP_ARCHIVE_GET  => "archive-get",
    OP_ARCHIVE_PUSH => "archive-push",
    OP_ARCHIVE_PULL => "archive-pull",
    OP_BACKUP       => "backup",
-    OP_EXPIRE       => "expire",
+    OP_EXPIRE       => "expire"
 };

+####################################################################################################################################
+# Configuration constants - configuration sections and keys
+####################################################################################################################################
 use constant
 {
    CONFIG_SECTION_COMMAND        => "command",
@@ -44,6 +55,7 @@ use constant
    CONFIG_KEY_HARDLINK           => "hardlink",
    CONFIG_KEY_ARCHIVE_REQUIRED   => "archive-required",
    CONFIG_KEY_ARCHIVE_MAX_MB     => "archive-max-mb",
+    CONFIG_KEY_START_FAST         => "start_fast",

    CONFIG_KEY_LEVEL_FILE         => "level-file",
    CONFIG_KEY_LEVEL_CONSOLE      => "level-console",
@@ -56,7 +68,9 @@ use constant
    CONFIG_KEY_PSQL               => "psql"
 };

+####################################################################################################################################
 # Command line parameters
+####################################################################################################################################
 my $strConfigFile;      # Configuration file
 my $strStanza;          # Stanza in the configuration file to load
 my $strType;            # Type of backup: full, differential (diff), incremental (incr)
@@ -65,10 +79,12 @@ GetOptions ("config=s" => \$strConfigFile,
            "stanza=s" => \$strStanza,
            "type=s" => \$strType)
    or die("Error in command line arguments\n");
-    
-# Global variables
-my %oConfig;

+####################################################################################################################################
+# Global variables
+####################################################################################################################################
+my %oConfig;            # Configuration hash
+    
 ####################################################################################################################################
 # CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false)
 ####################################################################################################################################
@@ -154,7 +170,8 @@ if (!defined($strOperation))
    confess &log(ERROR, "operation is not defined");
 }

-if ($strOperation ne OP_ARCHIVE_PUSH &&
+if ($strOperation ne OP_ARCHIVE_GET &&
+    $strOperation ne OP_ARCHIVE_PUSH &&
    $strOperation ne OP_ARCHIVE_PULL &&
    $strOperation ne OP_BACKUP &&
    $strOperation ne OP_EXPIRE)
@@ -189,7 +206,51 @@ log_level_set(uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_FILE, true, "I
              uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_CONSOLE, true, "ERROR")));

 ####################################################################################################################################
-# ARCHIVE-PUSH Command
+# ARCHIVE-GET Command
+####################################################################################################################################
+if ($strOperation eq OP_ARCHIVE_GET)
+{
+    # Make sure the archive file is defined
+    if (!defined($ARGV[1]))
+    {
+        confess &log(ERROR, "archive file not provided - show usage");
+    }
+
+    # Make sure the destination file is defined
+    if (!defined($ARGV[2]))
+    {
+        confess &log(ERROR, "destination file not provided - show usage");
+    }
+
+    # Init the file object
+    my $oFile = pg_backrest_file->new
+    (
+        strStanza => $strStanza,
+        bNoCompression => true,
+        strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
+        strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
+        strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
+        strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, true)
+    );
+
+    # Init the backup object
+    backup_init
+    (
+        undef,
+        $oFile
+    );
+
+    # Info for the Postgres log
+    &log(INFO, "getting archive log " . $ARGV[1]);
+
+    # Get the archive file
+    archive_get($ARGV[1], $ARGV[2]);
+
+    exit 0;
+}
+
+####################################################################################################################################
+# ARCHIVE-PUSH and ARCHIVE-PULL Commands
 ####################################################################################################################################
 if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
 {
@@ -475,11 +536,10 @@ if ($strOperation eq OP_BACKUP)
        exit 0
    }

-    backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH));
+    backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH),
+           config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_START_FAST, true, "n") eq "y" ? true : false);

    $strOperation = OP_EXPIRE;
-    
-    sleep(30);

    lock_file_remove();
 }
--- a/pg_backrest_backup.pm
+++ b/pg_backrest_backup.pm
@@ -22,7 +22,8 @@ use pg_backrest_db;

 use Exporter qw(import);

-our @EXPORT = qw(backup_init backup_thread_kill archive_push archive_pull archive_compress backup backup_expire archive_list_get);
+our @EXPORT = qw(backup_init backup_thread_kill archive_push archive_pull archive_get archive_compress
+                 backup backup_expire archive_list_get);

 my $oDb;
 my $oFile;
@@ -211,6 +212,29 @@ sub backup_thread_complete
    return true;
 }

+####################################################################################################################################
+# ARCHIVE_GET
+####################################################################################################################################
+sub archive_get
+{
+    my $strSourceArchive = shift;
+    my $strDestinationFile = shift;
+
+    my $strArchivePath = dirname($oFile->path_get(PATH_BACKUP_ARCHIVE, $strSourceArchive));
+
+    my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath,
+        "^${strSourceArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$");
+
+    if (scalar @stryArchiveFile != 1)
+    {
+        confess &log(ERROR, (scalar @stryArchiveFile) . " archive file(s) found for ${strSourceArchive}"); 
+    }
+
+    &log(DEBUG, "archive_get: cp ${stryArchiveFile[0]} ${strDestinationFile}");
+
+    $oFile->file_copy(PATH_BACKUP_ARCHIVE, $stryArchiveFile[0], PATH_DB_ABSOLUTE, $strDestinationFile);
+}
+
 ####################################################################################################################################
 # ARCHIVE_PUSH
 ####################################################################################################################################
@@ -1001,10 +1025,10 @@ sub backup_file
    my $fThreadFileLargeSizeMax = $lFileLargeSize / $iThreadLocalMax;

    &log(INFO, "file total ${lFileTotal}");
-    &log(INFO, "file small total ${lFileSmallTotal}, small size: " . file_size_format($lFileSmallSize) .
-         ", small thread avg total " . file_size_format(int($iThreadFileSmallTotalMax)));
-    &log(INFO, "file large total ${lFileLargeTotal}, large size: " . file_size_format($lFileLargeSize) .
-         ", large thread avg size " . file_size_format(int($fThreadFileLargeSizeMax)));
+    &log(DEBUG, "file small total ${lFileSmallTotal}, small size: " . file_size_format($lFileSmallSize) .
+                ", small thread avg total " . file_size_format(int($iThreadFileSmallTotalMax)));
+    &log(DEBUG, "file large total ${lFileLargeTotal}, large size: " . file_size_format($lFileLargeSize) .
+                ", large thread avg size " . file_size_format(int($fThreadFileLargeSizeMax)));

    foreach my $strFile (sort (keys %oFileCopyMap))
    {
@@ -1044,10 +1068,10 @@ sub backup_file
    for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++)
    {
        # Output info about how much work each thread is going to do
-        &log(INFO, "thread ${iThreadIdx} large total $oyThreadData[$iThreadIdx]{large_total}, " . 
-                   "size $oyThreadData[$iThreadIdx]{large_size}");
-        &log(INFO, "thread ${iThreadIdx} small total $oyThreadData[$iThreadIdx]{small_total}, " . 
-                   "size $oyThreadData[$iThreadIdx]{small_size}");
+        &log(DEBUG, "thread ${iThreadIdx} large total $oyThreadData[$iThreadIdx]{large_total}, " . 
+                    "size $oyThreadData[$iThreadIdx]{large_size}");
+        &log(DEBUG, "thread ${iThreadIdx} small total $oyThreadData[$iThreadIdx]{small_total}, " . 
+                    "size $oyThreadData[$iThreadIdx]{small_size}");

        # End each queue
        $oThreadQueue[$iThreadIdx]->enqueue(undef);
@@ -1192,6 +1216,7 @@ sub backup_file_thread
 sub backup
 {
    my $strDbClusterPath = shift;
+    my $bStartFast = shift;

    # Not supporting remote backup hosts yet
    if ($oFile->is_remote(PATH_BACKUP))
@@ -1260,7 +1285,7 @@ sub backup
    my %oBackupManifest;
    ${oBackupManifest}{backup}{label} = $strBackupPath;

-    my $strArchiveStart = $oDb->backup_start($strBackupPath);
+    my $strArchiveStart = $oDb->backup_start($strBackupPath, $bStartFast);
    ${oBackupManifest}{backup}{"archive-start"} = $strArchiveStart;

    &log(INFO, 'archive start: ' . ${oBackupManifest}{backup}{"archive-start"});
@@ -1440,8 +1465,6 @@ sub backup_expire

        while (defined($stryPath[$iIndex]))
        {
-            &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]);
-
            # Delete all backups that depend on the full backup.  Done in reverse order so that remaining backups will still
            # be consistent if the process dies
            foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse"))
@@ -1449,6 +1472,8 @@ sub backup_expire
                system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}");
            }
        
+            &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]);
+
            $iIndex++;
        }
    }
--- a/pg_backrest_db.pm
+++ b/pg_backrest_db.pm
@@ -126,9 +126,11 @@ sub backup_start
 {
    my $self = shift;
    my $strLabel = shift;
+    my $bStartFast = shift;

    return trim($self->psql_execute("set client_min_messages = 'warning';" . 
-                                    "copy (select pg_xlogfile_name(xlog) from pg_start_backup('${strLabel}') as xlog) to stdout"));
+                                    "copy (select pg_xlogfile_name(xlog) from pg_start_backup('${strLabel}'" .
+                                    ($bStartFast ? ", true" : "") . ") as xlog) to stdout"));
 }

 ####################################################################################################################################
--- a/pg_backrest_file.pm
+++ b/pg_backrest_file.pm
@@ -170,7 +170,7 @@ sub path_get
    my $bTemp = shift;      # Return the temp file for this path type - only some types have temp files

    # Only allow temp files for PATH_BACKUP_ARCHIVE and PATH_BACKUP_TMP
-    if (defined($bTemp) && $bTemp && !($strType eq PATH_BACKUP_ARCHIVE || $strType eq PATH_BACKUP_TMP))
+    if (defined($bTemp) && $bTemp && !($strType eq PATH_BACKUP_ARCHIVE || $strType eq PATH_BACKUP_TMP || $strType eq PATH_DB_ABSOLUTE))
    {
        confess &log(ASSERT, "temp file not supported on path " . $strType);
    }
@@ -178,6 +178,11 @@ sub path_get
    # Get absolute db path
    if ($strType eq PATH_DB_ABSOLUTE)
    {
+        if (defined($bTemp) && $bTemp)
+        {
+            return $strFile . ".backrest.tmp";
+        }
+
        return $strFile;
    }

@@ -738,7 +743,7 @@ sub file_compress

    if (!defined($self->{strCommandCompress}))
    {
-        confess &log(ASSERT, "\$strCommandChecksum not defined");
+        confess &log(ASSERT, "\$strCommandCompress not defined");
    }

    my $strPath = $self->path_get($strPathType, $strFile);
@@ -762,39 +767,42 @@ sub file_list_get
    my $strExpression = shift;
    my $strSortOrder = shift;

-    # For now this operation is not supported remotely.  Not currently needed.
+    # Get the root path for the file list
+    my $strPathList = $self->path_get($strPathType, $strPath);
+
+    # Builds the file list command
+#    my $strCommand = "ls ${strPathList} | egrep \"$strExpression\" 2> /dev/null";
+    my $strCommand = "ls -1 ${strPathList} 2> /dev/null";
+    
+    # Run the file list command
+    my $strFileList = "";
+
+    # Run remotely
    if ($self->is_remote($strPathType))
    {
-        confess &log(ASSERT, "remote operation not supported");
+        &log(TRACE, "file_list_get: remote ${strPathType}:${strPathList} ${strCommand}");
+
+        my $oSSH = $self->remote_get($strPathType);
+        $strFileList = $oSSH->capture($strCommand);
    }
-
-    my $strPathList = $self->path_get($strPathType, $strPath);
-    my $hDir;
-
-    opendir $hDir, $strPathList or confess &log(ERROR, "unable to open path ${strPathList}");
-    my @stryFileAll = readdir $hDir or confess &log(ERROR, "unable to get files for path ${strPathList}, expression ${strExpression}");
-    close $hDir;
-
-    my @stryFile;
-
-    if (@stryFileAll)
+    # Run locally
+    else
    {
-        @stryFile = grep(/$strExpression/i, @stryFileAll)
+        &log(TRACE, "file_list_get: local ${strPathType}:${strPathList} ${strCommand}");
+        $strFileList = capture($strCommand) or confess("error in ${strCommand}");
    }

-    if (@stryFile)
+    # Split the files into an array
+    my @stryFileList = grep(/$strExpression/i, split(/\n/, $strFileList));
+
+    # Return the array in reverse order if specified
+    if (defined($strSortOrder) && $strSortOrder eq "reverse")
    {
-        if (defined($strSortOrder) && $strSortOrder eq "reverse")
-        {
-            return sort {$b cmp $a} @stryFile;
-        }
-        else
-        {
-            return sort @stryFile;
-        }
+        return sort {$b cmp $a} @stryFileList;
    }
-
-    return @stryFile;
+    
+    # Return in normal sorted order
+    return sort @stryFileList;
 }

 ####################################################################################################################################