diff --git a/.gitignore b/.gitignore index ebec8ba8c..2a13959aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ **/*~ *~ + +*.swp diff --git a/README.md b/README.md index 87ab8dbbc..73b9c0cfe 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,24 @@ pg_backrest Simple Postgres Backup and Restore +planned for next release +======================== + +* Default restore.conf is written to each backup. + +* Able to set timeout on ssh connection in config file. + release notes ============= +v0.15: Added archive-get + +* Added archive-get functionality to aid in restores. + +* Added option to force a checkpoint when starting the backup (start_fast=y). + +------------- + v0.11: Minor fixes Tweaking a few settings after running backups for about a month. @@ -14,6 +29,8 @@ Tweaking a few settings after running backups for about a month. * Changed lock file conflicts on backup and expire commands to ERROR. They were set to DEBUG due to a copy-and-paste from the archive locks. +------------- + v0.10: Backup and archiving are functional This version has been put into production at Resonate, so it does work, but there are a number of major caveats. diff --git a/pg_backrest.conf b/pg_backrest.conf index ed78c12eb..1e84abbd1 100644 --- a/pg_backrest.conf +++ b/pg_backrest.conf @@ -18,6 +18,7 @@ path=/Users/backrest/test archive-required=y thread-max=2 thread-timeout=900 +start_fast=y [global:archive] path=/Users/dsteele/test diff --git a/pg_backrest.pl b/pg_backrest.pl index 16d278f0f..ad7078766 100755 --- a/pg_backrest.pl +++ b/pg_backrest.pl @@ -1,9 +1,14 @@ #!/usr/bin/perl +#################################################################################################################################### +# pg_backrest.pl - Simple Postgres Backup and Restore +#################################################################################################################################### -use threads; - +#################################################################################################################################### +# Perl includes +#################################################################################################################################### use strict; use warnings; +use threads; use File::Basename; use Getopt::Long; @@ -16,15 +21,21 @@ use pg_backrest_file; use pg_backrest_backup; use pg_backrest_db; -# Operation constants +#################################################################################################################################### +# Operation constants - basic operations that are allowed in backrest +#################################################################################################################################### use constant { + OP_ARCHIVE_GET => "archive-get", OP_ARCHIVE_PUSH => "archive-push", OP_ARCHIVE_PULL => "archive-pull", OP_BACKUP => "backup", - OP_EXPIRE => "expire", + OP_EXPIRE => "expire" }; +#################################################################################################################################### +# Configuration constants - configuration sections and keys +#################################################################################################################################### use constant { CONFIG_SECTION_COMMAND => "command", @@ -44,6 +55,7 @@ use constant CONFIG_KEY_HARDLINK => "hardlink", CONFIG_KEY_ARCHIVE_REQUIRED => "archive-required", CONFIG_KEY_ARCHIVE_MAX_MB => "archive-max-mb", + CONFIG_KEY_START_FAST => "start_fast", CONFIG_KEY_LEVEL_FILE => "level-file", CONFIG_KEY_LEVEL_CONSOLE => "level-console", @@ -56,7 +68,9 @@ use constant CONFIG_KEY_PSQL => "psql" }; +#################################################################################################################################### # Command line parameters +#################################################################################################################################### my $strConfigFile; # Configuration file my $strStanza; # Stanza in the configuration file to load my $strType; # Type of backup: full, differential (diff), incremental (incr) @@ -65,10 +79,12 @@ GetOptions ("config=s" => \$strConfigFile, "stanza=s" => \$strStanza, "type=s" => \$strType) or die("Error in command line arguments\n"); - -# Global variables -my %oConfig; +#################################################################################################################################### +# Global variables +#################################################################################################################################### +my %oConfig; # Configuration hash + #################################################################################################################################### # CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false) #################################################################################################################################### @@ -154,7 +170,8 @@ if (!defined($strOperation)) confess &log(ERROR, "operation is not defined"); } -if ($strOperation ne OP_ARCHIVE_PUSH && +if ($strOperation ne OP_ARCHIVE_GET && + $strOperation ne OP_ARCHIVE_PUSH && $strOperation ne OP_ARCHIVE_PULL && $strOperation ne OP_BACKUP && $strOperation ne OP_EXPIRE) @@ -189,7 +206,51 @@ log_level_set(uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_FILE, true, "I uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_CONSOLE, true, "ERROR"))); #################################################################################################################################### -# ARCHIVE-PUSH Command +# ARCHIVE-GET Command +#################################################################################################################################### +if ($strOperation eq OP_ARCHIVE_GET) +{ + # Make sure the archive file is defined + if (!defined($ARGV[1])) + { + confess &log(ERROR, "archive file not provided - show usage"); + } + + # Make sure the destination file is defined + if (!defined($ARGV[2])) + { + confess &log(ERROR, "destination file not provided - show usage"); + } + + # Init the file object + my $oFile = pg_backrest_file->new + ( + strStanza => $strStanza, + bNoCompression => true, + strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER), + strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST), + strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true), + strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, true) + ); + + # Init the backup object + backup_init + ( + undef, + $oFile + ); + + # Info for the Postgres log + &log(INFO, "getting archive log " . $ARGV[1]); + + # Get the archive file + archive_get($ARGV[1], $ARGV[2]); + + exit 0; +} + +#################################################################################################################################### +# ARCHIVE-PUSH and ARCHIVE-PULL Commands #################################################################################################################################### if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL) { @@ -475,11 +536,10 @@ if ($strOperation eq OP_BACKUP) exit 0 } - backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH)); + backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH), + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_START_FAST, true, "n") eq "y" ? true : false); $strOperation = OP_EXPIRE; - - sleep(30); lock_file_remove(); } diff --git a/pg_backrest_backup.pm b/pg_backrest_backup.pm index 12dc92b30..c5a0eed97 100644 --- a/pg_backrest_backup.pm +++ b/pg_backrest_backup.pm @@ -22,7 +22,8 @@ use pg_backrest_db; use Exporter qw(import); -our @EXPORT = qw(backup_init backup_thread_kill archive_push archive_pull archive_compress backup backup_expire archive_list_get); +our @EXPORT = qw(backup_init backup_thread_kill archive_push archive_pull archive_get archive_compress + backup backup_expire archive_list_get); my $oDb; my $oFile; @@ -211,6 +212,29 @@ sub backup_thread_complete return true; } +#################################################################################################################################### +# ARCHIVE_GET +#################################################################################################################################### +sub archive_get +{ + my $strSourceArchive = shift; + my $strDestinationFile = shift; + + my $strArchivePath = dirname($oFile->path_get(PATH_BACKUP_ARCHIVE, $strSourceArchive)); + + my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath, + "^${strSourceArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$"); + + if (scalar @stryArchiveFile != 1) + { + confess &log(ERROR, (scalar @stryArchiveFile) . " archive file(s) found for ${strSourceArchive}"); + } + + &log(DEBUG, "archive_get: cp ${stryArchiveFile[0]} ${strDestinationFile}"); + + $oFile->file_copy(PATH_BACKUP_ARCHIVE, $stryArchiveFile[0], PATH_DB_ABSOLUTE, $strDestinationFile); +} + #################################################################################################################################### # ARCHIVE_PUSH #################################################################################################################################### @@ -1001,10 +1025,10 @@ sub backup_file my $fThreadFileLargeSizeMax = $lFileLargeSize / $iThreadLocalMax; &log(INFO, "file total ${lFileTotal}"); - &log(INFO, "file small total ${lFileSmallTotal}, small size: " . file_size_format($lFileSmallSize) . - ", small thread avg total " . file_size_format(int($iThreadFileSmallTotalMax))); - &log(INFO, "file large total ${lFileLargeTotal}, large size: " . file_size_format($lFileLargeSize) . - ", large thread avg size " . file_size_format(int($fThreadFileLargeSizeMax))); + &log(DEBUG, "file small total ${lFileSmallTotal}, small size: " . file_size_format($lFileSmallSize) . + ", small thread avg total " . file_size_format(int($iThreadFileSmallTotalMax))); + &log(DEBUG, "file large total ${lFileLargeTotal}, large size: " . file_size_format($lFileLargeSize) . + ", large thread avg size " . file_size_format(int($fThreadFileLargeSizeMax))); foreach my $strFile (sort (keys %oFileCopyMap)) { @@ -1044,10 +1068,10 @@ sub backup_file for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++) { # Output info about how much work each thread is going to do - &log(INFO, "thread ${iThreadIdx} large total $oyThreadData[$iThreadIdx]{large_total}, " . - "size $oyThreadData[$iThreadIdx]{large_size}"); - &log(INFO, "thread ${iThreadIdx} small total $oyThreadData[$iThreadIdx]{small_total}, " . - "size $oyThreadData[$iThreadIdx]{small_size}"); + &log(DEBUG, "thread ${iThreadIdx} large total $oyThreadData[$iThreadIdx]{large_total}, " . + "size $oyThreadData[$iThreadIdx]{large_size}"); + &log(DEBUG, "thread ${iThreadIdx} small total $oyThreadData[$iThreadIdx]{small_total}, " . + "size $oyThreadData[$iThreadIdx]{small_size}"); # End each queue $oThreadQueue[$iThreadIdx]->enqueue(undef); @@ -1192,6 +1216,7 @@ sub backup_file_thread sub backup { my $strDbClusterPath = shift; + my $bStartFast = shift; # Not supporting remote backup hosts yet if ($oFile->is_remote(PATH_BACKUP)) @@ -1260,7 +1285,7 @@ sub backup my %oBackupManifest; ${oBackupManifest}{backup}{label} = $strBackupPath; - my $strArchiveStart = $oDb->backup_start($strBackupPath); + my $strArchiveStart = $oDb->backup_start($strBackupPath, $bStartFast); ${oBackupManifest}{backup}{"archive-start"} = $strArchiveStart; &log(INFO, 'archive start: ' . ${oBackupManifest}{backup}{"archive-start"}); @@ -1440,8 +1465,6 @@ sub backup_expire while (defined($stryPath[$iIndex])) { - &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]); - # Delete all backups that depend on the full backup. Done in reverse order so that remaining backups will still # be consistent if the process dies foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse")) @@ -1449,6 +1472,8 @@ sub backup_expire system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}"); } + &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]); + $iIndex++; } } diff --git a/pg_backrest_db.pm b/pg_backrest_db.pm index 69f6f6448..6f1ccb73a 100644 --- a/pg_backrest_db.pm +++ b/pg_backrest_db.pm @@ -126,9 +126,11 @@ sub backup_start { my $self = shift; my $strLabel = shift; + my $bStartFast = shift; return trim($self->psql_execute("set client_min_messages = 'warning';" . - "copy (select pg_xlogfile_name(xlog) from pg_start_backup('${strLabel}') as xlog) to stdout")); + "copy (select pg_xlogfile_name(xlog) from pg_start_backup('${strLabel}'" . + ($bStartFast ? ", true" : "") . ") as xlog) to stdout")); } #################################################################################################################################### diff --git a/pg_backrest_file.pm b/pg_backrest_file.pm index 59477484f..f1bfab11e 100644 --- a/pg_backrest_file.pm +++ b/pg_backrest_file.pm @@ -170,7 +170,7 @@ sub path_get my $bTemp = shift; # Return the temp file for this path type - only some types have temp files # Only allow temp files for PATH_BACKUP_ARCHIVE and PATH_BACKUP_TMP - if (defined($bTemp) && $bTemp && !($strType eq PATH_BACKUP_ARCHIVE || $strType eq PATH_BACKUP_TMP)) + if (defined($bTemp) && $bTemp && !($strType eq PATH_BACKUP_ARCHIVE || $strType eq PATH_BACKUP_TMP || $strType eq PATH_DB_ABSOLUTE)) { confess &log(ASSERT, "temp file not supported on path " . $strType); } @@ -178,6 +178,11 @@ sub path_get # Get absolute db path if ($strType eq PATH_DB_ABSOLUTE) { + if (defined($bTemp) && $bTemp) + { + return $strFile . ".backrest.tmp"; + } + return $strFile; } @@ -738,7 +743,7 @@ sub file_compress if (!defined($self->{strCommandCompress})) { - confess &log(ASSERT, "\$strCommandChecksum not defined"); + confess &log(ASSERT, "\$strCommandCompress not defined"); } my $strPath = $self->path_get($strPathType, $strFile); @@ -762,39 +767,42 @@ sub file_list_get my $strExpression = shift; my $strSortOrder = shift; - # For now this operation is not supported remotely. Not currently needed. + # Get the root path for the file list + my $strPathList = $self->path_get($strPathType, $strPath); + + # Builds the file list command +# my $strCommand = "ls ${strPathList} | egrep \"$strExpression\" 2> /dev/null"; + my $strCommand = "ls -1 ${strPathList} 2> /dev/null"; + + # Run the file list command + my $strFileList = ""; + + # Run remotely if ($self->is_remote($strPathType)) { - confess &log(ASSERT, "remote operation not supported"); + &log(TRACE, "file_list_get: remote ${strPathType}:${strPathList} ${strCommand}"); + + my $oSSH = $self->remote_get($strPathType); + $strFileList = $oSSH->capture($strCommand); } - - my $strPathList = $self->path_get($strPathType, $strPath); - my $hDir; - - opendir $hDir, $strPathList or confess &log(ERROR, "unable to open path ${strPathList}"); - my @stryFileAll = readdir $hDir or confess &log(ERROR, "unable to get files for path ${strPathList}, expression ${strExpression}"); - close $hDir; - - my @stryFile; - - if (@stryFileAll) + # Run locally + else { - @stryFile = grep(/$strExpression/i, @stryFileAll) + &log(TRACE, "file_list_get: local ${strPathType}:${strPathList} ${strCommand}"); + $strFileList = capture($strCommand) or confess("error in ${strCommand}"); } - if (@stryFile) + # Split the files into an array + my @stryFileList = grep(/$strExpression/i, split(/\n/, $strFileList)); + + # Return the array in reverse order if specified + if (defined($strSortOrder) && $strSortOrder eq "reverse") { - if (defined($strSortOrder) && $strSortOrder eq "reverse") - { - return sort {$b cmp $a} @stryFile; - } - else - { - return sort @stryFile; - } + return sort {$b cmp $a} @stryFileList; } - - return @stryFile; + + # Return in normal sorted order + return sort @stryFileList; } ####################################################################################################################################