mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2024-12-12 10:04:14 +02:00
d0b6f78b20
Master and standby can both be configured on the backup server and pgBackRest will automatically determine which is the master. This means no configuration changes for backup are required after failing over from a master to standby when a separate backup server is used.
413 lines
16 KiB
Perl
413 lines
16 KiB
Perl
####################################################################################################################################
|
|
# EXPIRE MODULE
|
|
####################################################################################################################################
|
|
package pgBackRest::Expire;
|
|
|
|
use threads;
|
|
use strict;
|
|
use warnings FATAL => qw(all);
|
|
use Carp qw(confess);
|
|
|
|
use Exporter qw(import);
|
|
use File::Basename qw(dirname);
|
|
use File::Path qw(remove_tree);
|
|
use Scalar::Util qw(looks_like_number);
|
|
|
|
use lib dirname($0);
|
|
use pgBackRest::Common::Exception;
|
|
use pgBackRest::Common::Log;
|
|
use pgBackRest::BackupCommon;
|
|
use pgBackRest::BackupInfo;
|
|
use pgBackRest::Config::Config;
|
|
use pgBackRest::File;
|
|
use pgBackRest::FileCommon;
|
|
use pgBackRest::Manifest;
|
|
use pgBackRest::Protocol::Common;
|
|
use pgBackRest::Protocol::Protocol;
|
|
|
|
####################################################################################################################################
|
|
# new
|
|
####################################################################################################################################
|
|
sub new
|
|
{
|
|
my $class = shift;
|
|
|
|
# Create the class hash
|
|
my $self = {};
|
|
bless $self, $class;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');
|
|
|
|
# Initialize file object
|
|
$self->{oFile} = new pgBackRest::File
|
|
(
|
|
optionGet(OPTION_STANZA),
|
|
optionGet(OPTION_REPO_PATH),
|
|
protocolGet(NONE)
|
|
);
|
|
|
|
# Initialize total archive expired
|
|
$self->{iArchiveExpireTotal} = 0;
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'self', value => $self}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# DESTROY
|
|
####################################################################################################################################
|
|
sub DESTROY
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my ($strOperation) = logDebugParam(__PACKAGE__ . '->DESTROY');
|
|
|
|
undef($self->{oFile});
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn($strOperation);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# logExpire
|
|
#
|
|
# Tracks which archive logs have been removed and provides log messages when needed.
|
|
####################################################################################################################################
|
|
sub logExpire
|
|
{
|
|
my $self = shift;
|
|
my $strArchiveFile = shift;
|
|
|
|
if (defined($strArchiveFile))
|
|
{
|
|
if (!defined($self->{strArchiveExpireStart}))
|
|
{
|
|
$self->{strArchiveExpireStart} = $strArchiveFile;
|
|
$self->{strArchiveExpireStop} = $strArchiveFile;
|
|
}
|
|
else
|
|
{
|
|
$self->{strArchiveExpireStop} = $strArchiveFile;
|
|
}
|
|
|
|
$self->{iArchiveExpireTotal}++;
|
|
}
|
|
else
|
|
{
|
|
if (defined($self->{strArchiveExpireStart}))
|
|
{
|
|
&log(DETAIL, 'remove archive: start = ' . substr($self->{strArchiveExpireStart}, 0, 24) .
|
|
', stop = ' . substr($self->{strArchiveExpireStop}, 0, 24));
|
|
}
|
|
|
|
undef($self->{strArchiveExpireStart});
|
|
}
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# process
|
|
#
|
|
# Removes expired backups and archive logs from the backup directory. Partial backups are not counted for expiration, so if full
|
|
# or differential retention is set to 2, there must be three complete backups before the oldest one can be deleted.
|
|
####################################################################################################################################
|
|
sub process
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my ($strOperation) = logDebugParam(__PACKAGE__ . '->process');
|
|
|
|
my @stryPath;
|
|
|
|
my $oFile = $self->{oFile};
|
|
my $strBackupClusterPath = $oFile->pathGet(PATH_BACKUP_CLUSTER);
|
|
my $iFullRetention = optionGet(OPTION_RETENTION_FULL, false);
|
|
my $iDifferentialRetention = optionGet(OPTION_RETENTION_DIFF, false);
|
|
my $strArchiveRetentionType = optionGet(OPTION_RETENTION_ARCHIVE_TYPE, false);
|
|
my $iArchiveRetention = optionGet(OPTION_RETENTION_ARCHIVE, false);
|
|
|
|
# Load or build backup.info
|
|
my $oBackupInfo = new pgBackRest::BackupInfo($oFile->pathGet(PATH_BACKUP_CLUSTER));
|
|
|
|
# Find all the expired full backups
|
|
if (defined($iFullRetention))
|
|
{
|
|
# Make sure iFullRetention is valid
|
|
if (!looks_like_number($iFullRetention) || $iFullRetention < 1)
|
|
{
|
|
confess &log(ERROR, 'retention-full must be a number >= 1');
|
|
}
|
|
|
|
@stryPath = $oBackupInfo->list(backupRegExpGet(true));
|
|
|
|
if (@stryPath > $iFullRetention)
|
|
{
|
|
# Expire all backups that depend on the full backup
|
|
for (my $iFullIdx = 0; $iFullIdx < @stryPath - $iFullRetention; $iFullIdx++)
|
|
{
|
|
my @stryRemoveList;
|
|
|
|
foreach my $strPath ($oBackupInfo->list('^' . $stryPath[$iFullIdx] . '.*'))
|
|
{
|
|
$oFile->remove(PATH_BACKUP_CLUSTER, "${strPath}/" . FILE_MANIFEST);
|
|
$oBackupInfo->delete($strPath);
|
|
|
|
if ($strPath ne $stryPath[$iFullIdx])
|
|
{
|
|
push(@stryRemoveList, $strPath);
|
|
}
|
|
}
|
|
|
|
&log(INFO, 'expire full backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iFullIdx] .
|
|
(@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : ''));
|
|
}
|
|
}
|
|
}
|
|
|
|
# Find all the expired differential backups
|
|
if (defined($iDifferentialRetention))
|
|
{
|
|
# Make sure iDifferentialRetention is valid
|
|
if (!looks_like_number($iDifferentialRetention) || $iDifferentialRetention < 1)
|
|
{
|
|
confess &log(ERROR, 'retention-diff must be a number >= 1');
|
|
}
|
|
|
|
@stryPath = $oBackupInfo->list(backupRegExpGet(false, true));
|
|
|
|
if (@stryPath > $iDifferentialRetention)
|
|
{
|
|
for (my $iDiffIdx = 0; $iDiffIdx < @stryPath - $iDifferentialRetention; $iDiffIdx++)
|
|
{
|
|
# Get a list of all differential and incremental backups
|
|
my @stryRemoveList;
|
|
|
|
foreach my $strPath ($oBackupInfo->list(backupRegExpGet(false, true, true)))
|
|
{
|
|
logDebugMisc($strOperation, "checking ${strPath} for differential expiration");
|
|
|
|
# Remove all differential and incremental backups before the oldest valid differential
|
|
if ($strPath lt $stryPath[$iDiffIdx + 1])
|
|
{
|
|
$oFile->remove(PATH_BACKUP_CLUSTER, "/${strPath}" . FILE_MANIFEST);
|
|
$oBackupInfo->delete($strPath);
|
|
|
|
if ($strPath ne $stryPath[$iDiffIdx])
|
|
{
|
|
push(@stryRemoveList, $strPath);
|
|
}
|
|
}
|
|
}
|
|
|
|
&log(INFO, 'expire diff backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iDiffIdx] .
|
|
(@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : ''));
|
|
}
|
|
}
|
|
}
|
|
|
|
$oBackupInfo->save();
|
|
|
|
# Remove backups from disk
|
|
foreach my $strBackup ($oFile->list(PATH_BACKUP_CLUSTER, undef, backupRegExpGet(true, true, true), 'reverse'))
|
|
{
|
|
if (!$oBackupInfo->current($strBackup))
|
|
{
|
|
&log(INFO, "remove expired backup ${strBackup}");
|
|
|
|
remove_tree("${strBackupClusterPath}/${strBackup}") > 0
|
|
or confess &log(ERROR, "unable to remove backup ${strBackup}", ERROR_PATH_REMOVE);
|
|
}
|
|
}
|
|
|
|
# Default archive retention if not explicily set
|
|
if (defined($iFullRetention))
|
|
{
|
|
if (!defined($iArchiveRetention))
|
|
{
|
|
$iArchiveRetention = $iFullRetention;
|
|
}
|
|
|
|
if (!defined($strArchiveRetentionType))
|
|
{
|
|
$strArchiveRetentionType = BACKUP_TYPE_FULL;
|
|
}
|
|
}
|
|
|
|
# If no archive retention type is set then exit
|
|
if (!defined($strArchiveRetentionType))
|
|
{
|
|
&log(INFO, 'archive retention type not set - archive logs will not be expired');
|
|
}
|
|
else
|
|
{
|
|
# Determine which backup type to use for archive retention (full, differential, incremental)
|
|
if ($strArchiveRetentionType eq BACKUP_TYPE_FULL)
|
|
{
|
|
@stryPath = $oBackupInfo->list(backupRegExpGet(true), 'reverse');
|
|
}
|
|
elsif ($strArchiveRetentionType eq BACKUP_TYPE_DIFF)
|
|
{
|
|
@stryPath = $oBackupInfo->list(backupRegExpGet(true, true), 'reverse');
|
|
}
|
|
elsif ($strArchiveRetentionType eq BACKUP_TYPE_INCR)
|
|
{
|
|
@stryPath = $oBackupInfo->list(backupRegExpGet(true, true, true), 'reverse');
|
|
}
|
|
|
|
# if no backups were found then preserve current archive logs - too soon to expire them
|
|
my $iBackupTotal = scalar @stryPath;
|
|
|
|
if ($iBackupTotal > 0)
|
|
{
|
|
# See if enough backups exist for expiration to start
|
|
my $strArchiveRetentionBackup = $stryPath[$iArchiveRetention - 1];
|
|
|
|
if (!defined($strArchiveRetentionBackup))
|
|
{
|
|
if ($strArchiveRetentionType eq BACKUP_TYPE_FULL && scalar @stryPath > 0)
|
|
{
|
|
&log(INFO, "full backup total < ${iArchiveRetention} - using oldest full backup for archive retention");
|
|
$strArchiveRetentionBackup = $stryPath[scalar @stryPath - 1];
|
|
}
|
|
}
|
|
|
|
# If a backup has been selected for retention then continue
|
|
if (defined($strArchiveRetentionBackup))
|
|
{
|
|
my $bRemove;
|
|
|
|
# Only expire if the selected backup has archive info - backups performed with --no-online will
|
|
# not have archive info and cannot be used for expiration.
|
|
if ($oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT,
|
|
$strArchiveRetentionBackup, INFO_BACKUP_KEY_ARCHIVE_START))
|
|
{
|
|
# Get archive info
|
|
my $oArchive = new pgBackRest::Archive();
|
|
my $strArchiveId = $oArchive->getArchiveId($oFile);
|
|
|
|
my $strArchiveExpireMax;
|
|
|
|
# Get archive ranges to preserve. Because archive retention can be less than total retention it is
|
|
# important to preserve older archive that is required to make the older backups consistent even though
|
|
# they cannot be played any further forward with PITR.
|
|
my @oyArchiveRange;
|
|
|
|
foreach my $strBackup ($oBackupInfo->list())
|
|
{
|
|
if ($strBackup le $strArchiveRetentionBackup &&
|
|
$oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_ARCHIVE_START))
|
|
{
|
|
my $oArchiveRange = {};
|
|
|
|
$$oArchiveRange{start} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT,
|
|
$strBackup, INFO_BACKUP_KEY_ARCHIVE_START);
|
|
|
|
if ($strBackup ne $strArchiveRetentionBackup)
|
|
{
|
|
$$oArchiveRange{stop} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT,
|
|
$strBackup, INFO_BACKUP_KEY_ARCHIVE_STOP);
|
|
}
|
|
else
|
|
{
|
|
$strArchiveExpireMax = $$oArchiveRange{start};
|
|
}
|
|
|
|
&log(DETAIL, "archive retention on backup ${strBackup}, start = $$oArchiveRange{start}" .
|
|
(defined($$oArchiveRange{stop}) ? ", stop = $$oArchiveRange{stop}" : ''));
|
|
|
|
push(@oyArchiveRange, $oArchiveRange);
|
|
}
|
|
}
|
|
|
|
# Get all major archive paths (timeline and first 64 bits of LSN)
|
|
foreach my $strPath ($oFile->list(PATH_BACKUP_ARCHIVE, $strArchiveId, "^[0-F]{16}\$"))
|
|
{
|
|
logDebugMisc($strOperation, "found major WAL path: ${strPath}");
|
|
$bRemove = true;
|
|
|
|
# Keep the path if it falls in the range of any backup in retention
|
|
foreach my $oArchiveRange (@oyArchiveRange)
|
|
{
|
|
if ($strPath ge substr($$oArchiveRange{start}, 0, 16) &&
|
|
(!defined($$oArchiveRange{stop}) || $strPath le substr($$oArchiveRange{stop}, 0, 16)))
|
|
{
|
|
$bRemove = false;
|
|
last;
|
|
}
|
|
}
|
|
|
|
# Remove the entire directory if all archive is expired
|
|
if ($bRemove)
|
|
{
|
|
my $strFullPath = $oFile->pathGet(PATH_BACKUP_ARCHIVE, $strArchiveId) . "/${strPath}";
|
|
|
|
remove_tree($strFullPath) > 0
|
|
or confess &log(ERROR, "unable to remove ${strFullPath}", ERROR_PATH_REMOVE);
|
|
|
|
# Log expire info
|
|
logDebugMisc($strOperation, "remove major WAL path: ${strFullPath}");
|
|
$self->logExpire($strPath);
|
|
}
|
|
# Else delete individual files instead if the major path is less than or equal to the most recent retention
|
|
# backup. This optimization prevents scanning though major paths that could not possibly have anything to
|
|
# expire.
|
|
elsif ($strPath le substr($strArchiveExpireMax, 0, 16))
|
|
{
|
|
# Look for files in the archive directory
|
|
foreach my $strSubPath ($oFile->list(PATH_BACKUP_ARCHIVE,
|
|
"${strArchiveId}/${strPath}", "^[0-F]{24}.*\$"))
|
|
{
|
|
$bRemove = true;
|
|
|
|
# Determine if the individual archive log is used in a backup
|
|
foreach my $oArchiveRange (@oyArchiveRange)
|
|
{
|
|
if (substr($strSubPath, 0, 24) ge $$oArchiveRange{start} &&
|
|
(!defined($$oArchiveRange{stop}) || substr($strSubPath, 0, 24) le $$oArchiveRange{stop}))
|
|
{
|
|
$bRemove = false;
|
|
last;
|
|
}
|
|
}
|
|
|
|
# Remove archive log if it is not used in a backup
|
|
if ($bRemove)
|
|
{
|
|
fileRemove($oFile->pathGet(PATH_BACKUP_ARCHIVE, "${strArchiveId}/${strSubPath}"));
|
|
|
|
logDebugMisc($strOperation, "remove WAL segment: ${strSubPath}");
|
|
|
|
# Log expire info
|
|
$self->logExpire(substr($strSubPath, 0, 24));
|
|
}
|
|
else
|
|
{
|
|
# Log that the file was not expired
|
|
$self->logExpire();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Log if no archive was expired
|
|
if ($self->{iArchiveExpireTotal} == 0)
|
|
{
|
|
&log(DETAIL, 'no archive to remove');
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn($strOperation);
|
|
}
|
|
|
|
1;
|