db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-09-16 09:06:18 +02:00
Code Issues Releases Activity

More robust config. Retention is read from config.

Browse Source
This commit is contained in:
David Steele
2014-02-06 12:49:54 -05:00
parent a2b0d7a674
commit 9d08f2a644
5 changed files with 191 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pg_backrest.conf
View File

@@ -1,22 +1,26 @@
[command]
[global:command]
#compress=pigz --rsyncable --best --stdout %file% # Ubuntu Linux
compress=/usr/bin/gzip --stdout %file%
decompress=/usr/bin/gzip -dc %file%
#checksum=sha1sum %file% | awk '{print \$1}' # Ubuntu Linux
checksum=/usr/bin/shasum %file% | awk '{print $1}'
manifest=/opt/local/bin/gfind %path% -printf '%P\t%y\t%u\t%g\t%m\t%T@\t%i\t%s\t%l\n'
psql=/Library/PostgreSQL/9.3/bin/psql -X --port=6001
psql=/Library/PostgreSQL/9.3/bin/psql -X %option%
[backup]
[global:backup]
user=backrest
host=localhost
path=/Users/backrest/test
[retention]
[global:retention]
full_retention=2
differential_retention=2
wal_retention_type=full,diff,incr
wal_retention=2 # 0 means since the last type
archive_retention_type=full
archive_retention=2
[cluster:db]
[db]
psql_options=--cluster=9.3/main
path=/Users/dsteele/test/db/common
[db:command:option]
psql=--port=6001

159
pg_backrest.pl
View File

@@ -15,7 +15,7 @@ use pg_backrest_backup;
# Command line parameters
my $strConfigFile;
my $strCluster;
my $strStanza;
my $strType = "incremental"; # Type of backup: full, differential (diff), incremental (incr)
my $bHardLink;
my $bNoChecksum;
@@ -25,30 +25,61 @@ GetOptions ("no-compression" => \$bNoCompression,
"no-checksum" => \$bNoChecksum,
"hardlink" => \$bHardLink,
"config=s" => \$strConfigFile,
"cluster=s" => \$strCluster,
"stanza=s" => \$strStanza,
"type=s" => \$strType)
or die("Error in command line arguments\n");
# Global variables
my %oConfig;
####################################################################################################################################
# CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false)
####################################################################################################################################
sub config_load
{
my $oConfigRef = shift;
my $strSection = shift;
my $strKey = shift;
my $bRequired = shift;
# Default is that the key is not required
if (!defined($bRequired))
{
$bRequired = 1;
$bRequired = false;
}
my $strValue = ${$oConfigRef}{"${strSection}"}{"${strKey}"};
my $strValue;
if ($bRequired && !defined($strValue))
# Look in the default stanza section
if ($strSection eq "stanza")
{
confess &log(ERROR, 'config value ${strSection}->${strKey} is undefined');
$strValue = $oConfig{"${strStanza}"}{"${strKey}"};
}
# Else look in the supplied section
else
{
# First check the stanza section
$strValue = $oConfig{"${strStanza}:${strSection}"}{"${strKey}"};
# If the stanza section value is undefined then check global
if (!defined($strValue))
{
$strValue = $oConfig{"global:${strSection}"}{"${strKey}"};
}
}
if (!defined($strValue) && $bRequired)
{
confess &log(ERROR, "config value " . (defined($strSection) ? $strSection : "[stanza]") . "->${strKey} is undefined");
}
if ($strSection eq "command")
{
my $strOption = config_load("command:option", $strKey);
if (defined($strOption))
{
$strValue =~ s/\%option\%/${strOption}/g;
}
}
return $strValue;
@@ -57,6 +88,13 @@ sub config_load
####################################################################################################################################
# START MAIN
####################################################################################################################################
# Error if no operation is specified
if (@ARGV < 1)
{
confess "operation my be specified (backup, expire, archive_push, ...) - show usage";
}
# Get the command
my $strOperation = $ARGV[0];
my $strLogFile = "";
@@ -76,58 +114,43 @@ if (!defined($strConfigFile))
$strConfigFile = "/etc/pg_backrest.conf";
}
my %oConfig;
tie %oConfig, 'Config::IniFiles', (-file => $strConfigFile) or confess &log(ERROR, "unable to find config file ${strConfigFile}");
# Load and check the cluster
if (!defined($strCluster))
if (!defined($strStanza))
{
$strCluster = "db"; #!!! Modify to load cluster from conf if there is only one, else error
confess "a backup stanza must be specified - show usage";
}
#file_init_archive
#(
# $bNoCompression,
# config_load(\%oConfig, "command", "checksum", !$bNoChecksum),
# config_load(\%oConfig, "command", "compress", !$bNoCompression),
# config_load(\%oConfig, "command", "decompress", !$bNoCompression),
# $oConfig{backup}{user},
# $oConfig{backup}{host},
# $oConfig{backup}{path},
# $strCluster,
#);
####################################################################################################################################
# ARCHIVE-PUSH Command
####################################################################################################################################
if ($strOperation eq "archive-push")
{
# Run file_init_archive - this is the minimal config needed to run archiving
my $oFile = pg_backrest_file->new
(
bNoCompression => $bNoCompression,
strCommandChecksum => config_load(\%oConfig, "command", "checksum", !$bNoChecksum),
strCommandCompress => config_load(\%oConfig, "command", "compress", !$bNoCompression),
strCommandDecompress => config_load(\%oConfig, "command", "decompress", !$bNoCompression),
strBackupUser => $oConfig{backup}{user},
strBackupHost => $oConfig{backup}{host},
strBackupPath => $oConfig{backup}{path},
strCluster => $strCluster
);
# $oFile->build();
backup_init
(
$oFile
);
# archive-push command must have two arguments
if (@ARGV != 2)
{
confess "not enough arguments - show usage";
}
# Run file_init_archive - this is the minimal config needed to run archiving
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => $bNoCompression,
strBackupUser => config_load("backup", "user"),
strBackupHost => config_load("backup", "host"),
strBackupPath => config_load("backup", "path", true),
strCommandChecksum => config_load("command", "checksum", !$bNoChecksum),
strCommandCompress => config_load("command", "compress", !$bNoCompression),
strCommandDecompress => config_load("command", "decompress", !$bNoCompression)
);
backup_init
(
$oFile
);
# Call the archive function
archive_push($ARGV[1]);
@@ -156,30 +179,20 @@ if ($strType ne "full" && $strType ne "differential" && $strType ne "incremental
# Run file_init_archive - the rest of the file config required for backup and restore
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => $bNoCompression,
strCommandChecksum => config_load(\%oConfig, "command", "checksum", !$bNoChecksum),
strCommandCompress => config_load(\%oConfig, "command", "compress", !$bNoCompression),
strCommandDecompress => config_load(\%oConfig, "command", "decompress", !$bNoCompression),
strCommandManifest => config_load(\%oConfig, "command", "manifest"),
strCommandPsql => config_load(\%oConfig, "command", "psql"),
strBackupUser => $oConfig{backup}{user},
strBackupHost => $oConfig{backup}{host},
strBackupPath => $oConfig{backup}{path},
strCluster => $strCluster,
strDbUser => $oConfig{"cluster:$strCluster"}{user},
strDbHost => $oConfig{"cluster:$strCluster"}{host}
strBackupUser => config_load("backup", "user"),
strBackupHost => config_load("backup", "host"),
strBackupPath => config_load("backup", "path", true),
strDbUser => config_load("stanza", "user"),
strDbHost => config_load("stanza", "host"),
strCommandChecksum => config_load("command", "checksum", !$bNoChecksum),
strCommandCompress => config_load("command", "compress", !$bNoCompression),
strCommandDecompress => config_load("command", "decompress", !$bNoCompression),
strCommandManifest => config_load("command", "manifest"),
strCommandPsql => config_load("command", "psql")
);
#$oFile->build();
#file_init_backup
#(
# config_load(\%oConfig, "command", "manifest"),
# $pg_backrest_file::strCommandPsql = config_load(\%oConfig, "command", "psql"),
# $oConfig{"cluster:$strCluster"}{user},
# $oConfig{"cluster:$strCluster"}{host}
#);
# Run backup_init - parameters required for backup and restore operations
backup_init
(
@@ -194,6 +207,24 @@ backup_init
####################################################################################################################################
if ($strOperation eq "backup")
{
backup($oConfig{"cluster:$strCluster"}{path});
backup(config_load("stanza", "path"));
$strOperation = "expire";
}
####################################################################################################################################
# EXPIRE
####################################################################################################################################
if ($strOperation eq "expire")
{
backup_expire
(
$oFile->path_get(PATH_BACKUP_CLUSTER),
config_load("retention", "full_retention"),
config_load("retention", "differential_retention"),
config_load("retention", "archive_retention_type"),
config_load("retention", "archive_retention")
);
exit 0;
}

105
pg_backrest_backup.pm
View File

@@ -11,6 +11,7 @@ use Carp;
use File::Basename;
use File::Path;
use JSON;
use Scalar::Util qw(looks_like_number);
use lib dirname($0);
use pg_backrest_utility;
@@ -18,7 +19,7 @@ use pg_backrest_file;
use Exporter qw(import);
our @EXPORT = qw(backup_init archive_push backup);
our @EXPORT = qw(backup_init archive_push backup backup_expire);
my $oFile;
my $strType = "incremental"; # Type of backup: full, differential (diff), incremental (incr)
@@ -612,11 +613,8 @@ sub backup
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
# Rename the backup tmp path to complete the backup
&log(DEBUG, " moving ${strBackupTmpPath} to " . $oFile->path_get(PATH_BACKUP_CLUSTER, $strBackupPath));
$oFile->file_move(PATH_BACKUP_TMP, undef, PATH_BACKUP_CLUSTER, $strBackupPath);
# rename($strBackupTmpPath, "${pg_backrest_file::strBackupClusterPath}/${strBackupPath}") or confess &log(ERROR, "unable to ${strBackupTmpPath} rename to ${strBackupPath}");
# Expire backups (!!! Need to read this from config file)
backup_expire($oFile->path_get(PATH_BACKUP_CLUSTER), 2, 2, "full", 2);
}
####################################################################################################################################
@@ -667,6 +665,14 @@ sub archive_list_get
####################################################################################################################################
# BACKUP_EXPIRE
#
# Removes expired backups and archive logs from the backup directory.
#
# iFullRetention - Optional, must be greater than 0 when supplied. Defines the number of full backups that will be retained.
# Partial backups do not count, so if iFullRetention is set to 2, there must be 3 complete full backups before the oldest one can
# be deleted.
#
# iDifferentialRetention -
####################################################################################################################################
sub backup_expire
{
@@ -676,55 +682,98 @@ sub backup_expire
my $strArchiveRetentionType = shift; # Type of backup to base archive retention on
my $iArchiveRetention = shift; # Number of backups worth of archive to keep
# Find all the expired full backups
my $iIndex = $iFullRetention;
my $strPath;
my @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
while (defined($stryPath[$iIndex]))
my @stryPath;
# Find all the expired full backups
if (defined($iFullRetention))
{
&log(INFO, "removed expired full backup: " . $stryPath[$iIndex]);
# Delete all backups that depend on the full backup. Done in reverse order so that remaining backups will still
# be consistent if the process dies
foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse"))
# Make sure iFullRetention is valid
if (!looks_like_number($iFullRetention) || $iFullRetention < 1)
{
system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}");
confess &log(ERROR, "full_rentention must be a number >= 1");
}
my $iIndex = $iFullRetention;
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
while (defined($stryPath[$iIndex]))
{
&log(INFO, "removed expired full backup: " . $stryPath[$iIndex]);
# Delete all backups that depend on the full backup. Done in reverse order so that remaining backups will still
# be consistent if the process dies
foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse"))
{
system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}");
}
$iIndex++;
$iIndex++;
}
}
# Find all the expired differential backups
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse");
if (defined($stryPath[$iDifferentialRetention]))
if (defined($iDifferentialRetention))
{
# Get a list of all differential and incremental backups
foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse"))
# Make sure iDifferentialRetention is valid
if (!looks_like_number($iDifferentialRetention) || $iDifferentialRetention < 1)
{
# Remove all differential and incremental backups before the oldest valid differential
if (substr($strPath, 0, length($strPath) - 1) lt $stryPath[$iDifferentialRetention])
confess &log(ERROR, "differential_rentention must be a number >= 1");
}
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse");
if (defined($stryPath[$iDifferentialRetention]))
{
# Get a list of all differential and incremental backups
foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse"))
{
system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}");
&log(INFO, "removed expired diff/incr backup ${strPath}");
# Remove all differential and incremental backups before the oldest valid differential
if (substr($strPath, 0, length($strPath) - 1) lt $stryPath[$iDifferentialRetention])
{
system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}");
&log(INFO, "removed expired diff/incr backup ${strPath}");
}
}
}
}
# If no archive retention type is set then exit
if (!defined($strArchiveRetentionType))
{
&log(INFO, "archive rentention type not set - archive logs will not be expired");
return;
}
# Determine which backup type to use for archive retention (full, differential, incremental)
if ($strArchiveRetentionType eq "full")
{
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
}
elsif ($strArchiveRetentionType eq "differential")
elsif ($strArchiveRetentionType eq "differential" || $strArchiveRetentionType eq "diff")
{
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 0), "reverse");
}
else
elsif ($strArchiveRetentionType eq "incremental" || $strArchiveRetentionType eq "incr")
{
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse");
}
else
{
confess &log(ERROR, "unknown archive_retention_type '${strArchiveRetentionType}'");
}
# Make sure that iArchiveRetention is set and valid
if (!defined($iArchiveRetention))
{
confess &log(ERROR, "archive_rentention must be set if archive_retention_type is set");
return;
}
if (!looks_like_number($iArchiveRetention) || $iArchiveRetention < 0)
{
confess &log(ERROR, "archive_rentention must be a number >= 0");
}
# if no backups were found then preserve current archive logs - too scary to delete them!
my $iBackupTotal = scalar @stryPath;

14
pg_backrest_file.pm
View File

@@ -44,7 +44,7 @@ has strBackupClusterPath => (is => 'bare'); # Backup cluster path
# Process flags
has bNoCompression => (is => 'bare');
has strCluster => (is => 'bare');
has strStanza => (is => 'bare');
sub BUILD
{
@@ -57,7 +57,7 @@ sub BUILD
}
# Create the backup cluster path
$self->{strBackupClusterPath} = $self->{strBackupPath} . "/" . $self->{strCluster};
$self->{strBackupClusterPath} = $self->{strBackupPath} . "/" . $self->{strStanza};
# Connect SSH object if backup host is defined
if (defined($self->{strBackupHost}))
@@ -154,15 +154,15 @@ sub path_get
}
# Make sure the cluster is defined
if (!defined($self->{strCluster}))
if (!defined($self->{strStanza}))
{
confess &log(ASSERT, "\$strCluster not yet defined");
confess &log(ASSERT, "\$strStanza not yet defined");
}
# Get the backup tmp path
if ($strType eq PATH_BACKUP_TMP)
{
my $strTempPath = "$self->{strBackupPath}/tmp/$self->{strCluster}.tmp";
my $strTempPath = "$self->{strBackupPath}/tmp/$self->{strStanza}.tmp";
if (defined($bTemp) && $bTemp)
{
@@ -175,7 +175,7 @@ sub path_get
# Get the backup archive path
if ($strType eq PATH_BACKUP_ARCHIVE)
{
my $strArchivePath = "$self->{strBackupPath}/archive/$self->{strCluster}";
my $strArchivePath = "$self->{strBackupPath}/archive/$self->{strStanza}";
my $strArchive;
if (defined($strFile))
@@ -194,7 +194,7 @@ sub path_get
if ($strType eq PATH_BACKUP_CLUSTER)
{
return $self->{strBackupPath} . "/backup/$self->{strCluster}" . (defined($strFile) ? "/${strFile}" : "");
return $self->{strBackupPath} . "/backup/$self->{strStanza}" . (defined($strFile) ? "/${strFile}" : "");
}
# Error when path type not recognized

2
test/test.pl
View File

@@ -101,7 +101,7 @@ sub archive_command_build
my $bCompression = shift;
my $bChecksum = shift;
my $strCommand = "$strBackRestBinPath/pg_backrest.pl --cluster=db --config=$strBackRestBinPath/pg_backrest.conf";
my $strCommand = "$strBackRestBinPath/pg_backrest.pl --stanza=db --config=$strBackRestBinPath/pg_backrest.conf";
if (!$bCompression)
{