1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
pgbackrest/pg_backrest.pl
David Steele 9d949b7ad6 v0.11: Minor fixes
Tweaking a few settings after running backups for about a month.

* Removed master_stderr_discard option on database SSH connections.  There have been occasional lockups and they could be related issues originally seen in the file code.

* Changed lock file conflicts on backup and expire commands to ERROR.  They were set to DEBUG due to a copy-and-paste from the archive locks.
2014-03-26 10:53:49 -04:00

514 lines
19 KiB
Perl
Executable File

#!/usr/bin/perl
use threads;
use strict;
use warnings;
use File::Basename;
use Getopt::Long;
use Config::IniFiles;
use Carp;
use lib dirname($0);
use pg_backrest_utility;
use pg_backrest_file;
use pg_backrest_backup;
use pg_backrest_db;
# Operation constants
use constant
{
OP_ARCHIVE_PUSH => "archive-push",
OP_ARCHIVE_PULL => "archive-pull",
OP_BACKUP => "backup",
OP_EXPIRE => "expire",
};
use constant
{
CONFIG_SECTION_COMMAND => "command",
CONFIG_SECTION_COMMAND_OPTION => "command:option",
CONFIG_SECTION_LOG => "log",
CONFIG_SECTION_BACKUP => "backup",
CONFIG_SECTION_ARCHIVE => "archive",
CONFIG_SECTION_RETENTION => "retention",
CONFIG_SECTION_STANZA => "stanza",
CONFIG_KEY_USER => "user",
CONFIG_KEY_HOST => "host",
CONFIG_KEY_PATH => "path",
CONFIG_KEY_THREAD_MAX => "thread-max",
CONFIG_KEY_THREAD_TIMEOUT => "thread-timeout",
CONFIG_KEY_HARDLINK => "hardlink",
CONFIG_KEY_ARCHIVE_REQUIRED => "archive-required",
CONFIG_KEY_ARCHIVE_MAX_MB => "archive-max-mb",
CONFIG_KEY_LEVEL_FILE => "level-file",
CONFIG_KEY_LEVEL_CONSOLE => "level-console",
CONFIG_KEY_COMPRESS => "compress",
CONFIG_KEY_COMPRESS_ASYNC => "compress-async",
CONFIG_KEY_DECOMPRESS => "decompress",
CONFIG_KEY_CHECKSUM => "checksum",
CONFIG_KEY_MANIFEST => "manifest",
CONFIG_KEY_PSQL => "psql"
};
# Command line parameters
my $strConfigFile; # Configuration file
my $strStanza; # Stanza in the configuration file to load
my $strType; # Type of backup: full, differential (diff), incremental (incr)
GetOptions ("config=s" => \$strConfigFile,
"stanza=s" => \$strStanza,
"type=s" => \$strType)
or die("Error in command line arguments\n");
# Global variables
my %oConfig;
####################################################################################################################################
# CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false)
####################################################################################################################################
sub config_load
{
my $strSection = shift;
my $strKey = shift;
my $bRequired = shift;
my $strDefault = shift;
# Default is that the key is not required
if (!defined($bRequired))
{
$bRequired = false;
}
my $strValue;
# Look in the default stanza section
if ($strSection eq CONFIG_SECTION_STANZA)
{
$strValue = $oConfig{"${strStanza}"}{"${strKey}"};
}
# Else look in the supplied section
else
{
# First check the stanza section
$strValue = $oConfig{"${strStanza}:${strSection}"}{"${strKey}"};
# If the stanza section value is undefined then check global
if (!defined($strValue))
{
$strValue = $oConfig{"global:${strSection}"}{"${strKey}"};
}
}
if (!defined($strValue) && $bRequired)
{
if (defined($strDefault))
{
return $strDefault;
}
confess &log(ERROR, "config value " . (defined($strSection) ? $strSection : "[stanza]") . "->${strKey} is undefined");
}
if ($strSection eq CONFIG_SECTION_COMMAND)
{
my $strOption = config_load(CONFIG_SECTION_COMMAND_OPTION, $strKey);
if (defined($strOption))
{
$strValue =~ s/\%option\%/${strOption}/g;
}
}
return $strValue;
}
####################################################################################################################################
# SAFE_EXIT - terminate all SSH sessions when the script is terminated
####################################################################################################################################
sub safe_exit
{
my $iTotal = backup_thread_kill();
confess &log(ERROR, "process was terminated on signal, ${iTotal} threads stopped");
}
$SIG{TERM} = \&safe_exit;
$SIG{HUP} = \&safe_exit;
$SIG{INT} = \&safe_exit;
####################################################################################################################################
# START MAIN
####################################################################################################################################
# Get the operation
my $strOperation = $ARGV[0];
# Validate the operation
if (!defined($strOperation))
{
confess &log(ERROR, "operation is not defined");
}
if ($strOperation ne OP_ARCHIVE_PUSH &&
$strOperation ne OP_ARCHIVE_PULL &&
$strOperation ne OP_BACKUP &&
$strOperation ne OP_EXPIRE)
{
confess &log(ERROR, "invalid operation ${strOperation}");
}
# Type should only be specified for backups
if (defined($strType) && $strOperation ne OP_BACKUP)
{
confess &log(ERROR, "type can only be specified for the backup operation")
}
####################################################################################################################################
# LOAD CONFIG FILE
####################################################################################################################################
if (!defined($strConfigFile))
{
$strConfigFile = "/etc/pg_backrest.conf";
}
tie %oConfig, 'Config::IniFiles', (-file => $strConfigFile) or confess &log(ERROR, "unable to find config file ${strConfigFile}");
# Load and check the cluster
if (!defined($strStanza))
{
confess "a backup stanza must be specified - show usage";
}
# Set the log levels
log_level_set(uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_FILE, true, "INFO")),
uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_CONSOLE, true, "ERROR")));
####################################################################################################################################
# ARCHIVE-PUSH Command
####################################################################################################################################
if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
{
# If an archive section has been defined, use that instead of the backup section when operation is OP_ARCHIVE_PUSH
my $strSection = defined(config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH)) ? CONFIG_SECTION_ARCHIVE : CONFIG_SECTION_BACKUP;
# Get the async compress flag. If compress_async=y then compression is off for the initial push
my $bCompressAsync = config_load($strSection, CONFIG_KEY_COMPRESS_ASYNC, true, "n") eq "n" ? false : true;
# Get the async compress flag. If compress_async=y then compression is off for the initial push
my $strStopFile;
my $strArchivePath;
# If logging locally then create the stop archiving file name
if ($strSection eq CONFIG_SECTION_ARCHIVE)
{
$strArchivePath = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH);
$strStopFile = "${strArchivePath}/lock/${strStanza}-archive.stop";
}
# Perform the archive-push
if ($strOperation eq OP_ARCHIVE_PUSH)
{
# Call the archive_push function
if (!defined($ARGV[1]))
{
confess &log(ERROR, "source archive file not provided - show usage");
}
# If the stop file exists then discard the archive log
if (defined($strStopFile))
{
if (-e $strStopFile)
{
&log(ERROR, "archive stop file exists ($strStopFile), discarding " . basename($ARGV[1]));
exit 0;
}
}
# Make sure that archive-push is running locally
if (defined(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST)))
{
confess &log(ERROR, "stanza host cannot be set on archive-push - must be run locally on db server");
}
# Get the compress flag
my $bCompress = $bCompressAsync ? false : config_load($strSection, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
# Get the checksum flag
my $bChecksum = config_load($strSection, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
# Run file_init_archive - this is the minimal config needed to run archiving
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load($strSection, CONFIG_KEY_USER),
strBackupHost => config_load($strSection, CONFIG_KEY_HOST),
strBackupPath => config_load($strSection, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum
);
&log(INFO, "pushing archive log " . $ARGV[1] . ($bCompressAsync ? " asynchronously" : ""));
archive_push($ARGV[1]);
# Only continue if we are archiving local and a backup server is defined
if (!($strSection eq CONFIG_SECTION_ARCHIVE && defined(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST))))
{
exit 0;
}
# Set the operation so that archive-pull will be called next
$strOperation = OP_ARCHIVE_PULL;
# fork and exit the parent process
if (fork())
{
exit 0;
}
}
# Perform the archive-pull
if ($strOperation eq OP_ARCHIVE_PULL)
{
# Make sure that archive-pull is running on the db server
if (defined(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST)))
{
confess &log(ERROR, "stanza host cannot be set on archive-pull - must be run locally on db server");
}
# Create a lock file to make sure archive-pull does not run more than once
my $strLockFile = "${strArchivePath}/lock/${strStanza}-archive.lock";
if (!lock_file_create($strLockFile))
{
&log(DEBUG, "archive-pull process is already running - exiting");
exit 0
}
# Build the basic command string that will be used to modify the command during processing
my $strCommand = $^X . " " . $0 . " --stanza=${strStanza}";
# Get the new operational flags
my $bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
my $bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
my $iArchiveMaxMB = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_ARCHIVE_MAX_MB);
eval
{
# Run file_init_archive - this is the minimal config needed to run archive pulling
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX),
undef,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT)
);
# Call the archive_pull function Continue to loop as long as there are files to process.
while (archive_pull($strArchivePath . "/archive/${strStanza}", $strStopFile, $strCommand, $iArchiveMaxMB))
{
}
};
# If there were errors above then start compressing
if ($@)
{
if ($bCompressAsync)
{
&log(ERROR, "error during transfer: $@");
&log(WARN, "errors during transfer, starting compression");
# Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => false,
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX),
undef,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT)
);
archive_compress($strArchivePath . "/archive/${strStanza}", $strCommand, 256);
}
else
{
confess $@;
}
}
lock_file_remove();
}
exit 0;
}
####################################################################################################################################
# OPEN THE LOG FILE
####################################################################################################################################
if (defined(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST)))
{
confess &log(ASSERT, "backup/expire operations must be performed locally on the backup server");
}
log_file_set(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . "/log/${strStanza}");
####################################################################################################################################
# GET MORE CONFIG INFO
####################################################################################################################################
# Set the backup type
if (!defined($strType))
{
$strType = "incremental";
}
elsif ($strType eq "diff")
{
$strType = "differential";
}
elsif ($strType eq "incr")
{
$strType = "incremental";
}
elsif ($strType ne "full" && $strType ne "differential" && $strType ne "incremental")
{
confess &log(ERROR, "backup type must be full, differential (diff), incremental (incr)");
}
# Get the operational flags
my $bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
my $bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
# Run file_init_archive - the rest of the file config required for backup and restore
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL)
);
my $oDb = pg_backrest_db->new
(
strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL),
oDbSSH => $oFile->{oDbSSH}
);
# Run backup_init - parameters required for backup and restore operations
backup_init
(
$oDb,
$oFile,
$strType,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HARDLINK, true, "n") eq "y" ? true : false,
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX),
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_ARCHIVE_REQUIRED, true, "y") eq "y" ? true : false,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT)
);
####################################################################################################################################
# BACKUP
####################################################################################################################################
if ($strOperation eq OP_BACKUP)
{
my $strLockFile = $oFile->path_get(PATH_BACKUP, "lock/${strStanza}-backup.lock");
if (!lock_file_create($strLockFile))
{
&log(ERROR, "backup process is already running for stanza ${strStanza} - exiting");
exit 0
}
backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH));
$strOperation = OP_EXPIRE;
sleep(30);
lock_file_remove();
}
####################################################################################################################################
# EXPIRE
####################################################################################################################################
if ($strOperation eq OP_EXPIRE)
{
my $strLockFile = $oFile->path_get(PATH_BACKUP, "lock/${strStanza}-expire.lock");
if (!lock_file_create($strLockFile))
{
&log(ERROR, "expire process is already running for stanza ${strStanza} - exiting");
exit 0
}
backup_expire
(
$oFile->path_get(PATH_BACKUP_CLUSTER),
config_load(CONFIG_SECTION_RETENTION, "full_retention"),
config_load(CONFIG_SECTION_RETENTION, "differential_retention"),
config_load(CONFIG_SECTION_RETENTION, "archive_retention_type"),
config_load(CONFIG_SECTION_RETENTION, "archive_retention")
);
lock_file_remove();
exit 0;
}
confess &log(ASSERT, "invalid operation ${strOperation} - missing handler block");