db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-06-23 00:07:44 +02:00
Code Issues Releases Activity

The expire command is implemented entirely in C.

Browse Source 
This implementation duplicates the functionality of the Perl code but does so with different logic and includes full unit tests.

Along the way at least one bug was fixed, see issue .

Contributed by Cynthia Shang.
This commit is contained in:
David Steele
2019-06-18 15:19:20 -04:00
parent f88bee7b33
commit 434cd83285
15 changed files with 1585 additions and 893 deletions

16
doc/xml/release.xml

@ -15,6 +15,18 @@
<release date="XXXX-XX-XX" version="2.15dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-improvement-list>
<release-item>
<release-item-contributor-list>
<release-item-contributor id="cynthia.shang"/>
</release-item-contributor-list>
<p>The <cmd>expire</cmd> command is implemented entirely in C.</p>
</release-item>
<release-item>
<p>The <cmd>local</cmd> command for restore is implemented entirely in C.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="julian.zhang"/>
@ -24,10 +36,6 @@
<p>Remove hard-coded <postgres/> user so <id>$PGUSER</id> works.</p>
</release-item>
<release-item>
<p>The <cmd>local</cmd> command for restore is implemented entirely in C.</p>
</release-item>
<release-item>
<p>Rename <br-option>repo-s3-verify-ssl</br-option> option to <br-option>repo-s3-verify-tls</br-option>.</p>

461
lib/pgBackRest/Expire.pm

@ -1,461 +0,0 @@
####################################################################################################################################
# EXPIRE MODULE
####################################################################################################################################
package pgBackRest::Expire;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
use File::Basename qw(dirname);
use Scalar::Util qw(looks_like_number);
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Backup::Common;
use pgBackRest::Backup::Info;
use pgBackRest::Config::Config;
use pgBackRest::InfoCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift;
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');
# Initialize total archive expired
$self->{iArchiveExpireTotal} = 0;
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# logExpire
#
# Tracks which archive logs have been removed and provides log messages when needed.
####################################################################################################################################
sub logExpire
{
my $self = shift;
my $strArchiveId = shift;
my $strArchiveFile = shift;
if (defined($strArchiveFile))
{
if (!defined($self->{strArchiveExpireStart}))
{
$self->{strArchiveExpireStart} = $strArchiveFile;
$self->{strArchiveExpireStop} = $strArchiveFile;
}
else
{
$self->{strArchiveExpireStop} = $strArchiveFile;
}
$self->{iArchiveExpireTotal}++;
}
else
{
if (defined($self->{strArchiveExpireStart}))
{
&log(DETAIL, "remove archive: archiveId = ${strArchiveId}, start = " . substr($self->{strArchiveExpireStart}, 0, 24) .
", stop = " . substr($self->{strArchiveExpireStop}, 0, 24));
}
undef($self->{strArchiveExpireStart});
}
}
####################################################################################################################################
# process
#
# Removes expired backups and archive logs from the backup directory. Partial backups are not counted for expiration, so if full
# or differential retention is set to 2, there must be three complete backups before the oldest one can be deleted.
####################################################################################################################################
sub process
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->process');
my @stryPath;
my $oStorageRepo = storageRepo();
my $strBackupClusterPath = $oStorageRepo->pathGet(STORAGE_REPO_BACKUP);
my $iFullRetention = cfgOption(CFGOPT_REPO_RETENTION_FULL, false);
my $iDifferentialRetention = cfgOption(CFGOPT_REPO_RETENTION_DIFF, false);
my $strArchiveRetentionType = cfgOption(CFGOPT_REPO_RETENTION_ARCHIVE_TYPE, false);
my $iArchiveRetention = cfgOption(CFGOPT_REPO_RETENTION_ARCHIVE, false);
# Load the backup.info
my $oBackupInfo = new pgBackRest::Backup::Info($oStorageRepo->pathGet(STORAGE_REPO_BACKUP));
# Find all the expired full backups
if (defined($iFullRetention))
{
# Make sure iFullRetention is valid
if (!looks_like_number($iFullRetention) || $iFullRetention < 1)
{
confess &log(ERROR, cfgOptionName(CFGOPT_REPO_RETENTION_FULL) . ' must be a number >= 1');
}
@stryPath = $oBackupInfo->list(backupRegExpGet(true));
if (@stryPath > $iFullRetention)
{
# Expire all backups that depend on the full backup
for (my $iFullIdx = 0; $iFullIdx < @stryPath - $iFullRetention; $iFullIdx++)
{
my @stryRemoveList;
foreach my $strPath ($oBackupInfo->list('^' . $stryPath[$iFullIdx] . '.*'))
{
$oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}/" . FILE_MANIFEST . INI_COPY_EXT);
$oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}/" . FILE_MANIFEST);
$oBackupInfo->delete($strPath);
if ($strPath ne $stryPath[$iFullIdx])
{
push(@stryRemoveList, $strPath);
}
}
&log(INFO, 'expire full backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iFullIdx] .
(@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : ''));
}
}
}
# Find all the expired differential backups
if (defined($iDifferentialRetention))
{
# Make sure iDifferentialRetention is valid
if (!looks_like_number($iDifferentialRetention) || $iDifferentialRetention < 1)
{
confess &log(ERROR, cfgOptionName(CFGOPT_REPO_RETENTION_DIFF) . ' must be a number >= 1');
}
# Get a list of full and differential backups. Full are considered differential for the purpose of retention.
# Example: F1, D1, D2, F2 and repo-retention-diff=2, then F1,D2,F2 will be retained, not D2 and D1 as might be expected.
@stryPath = $oBackupInfo->list(backupRegExpGet(true, true));
if (@stryPath > $iDifferentialRetention)
{
for (my $iDiffIdx = 0; $iDiffIdx < @stryPath - $iDifferentialRetention; $iDiffIdx++)
{
# Skip if this is a full backup. Full backups only count as differential when deciding which differential backups
# to expire.
next if ($stryPath[$iDiffIdx] =~ backupRegExpGet(true));
# Get a list of all differential and incremental backups
my @stryRemoveList;
foreach my $strPath ($oBackupInfo->list(backupRegExpGet(false, true, true)))
{
logDebugMisc($strOperation, "checking ${strPath} for differential expiration");
# Remove all differential and incremental backups before the oldest valid differential
if ($strPath lt $stryPath[$iDiffIdx + 1])
{
$oStorageRepo->remove(STORAGE_REPO_BACKUP . "/${strPath}" . FILE_MANIFEST);
$oBackupInfo->delete($strPath);
if ($strPath ne $stryPath[$iDiffIdx])
{
push(@stryRemoveList, $strPath);
}
}
}
&log(INFO, 'expire diff backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iDiffIdx] .
(@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : ''));
}
}
}
$oBackupInfo->save();
# Remove backups from disk
foreach my $strBackup ($oStorageRepo->list(
STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))
{
if (!$oBackupInfo->current($strBackup))
{
&log(INFO, "remove expired backup ${strBackup}");
$oStorageRepo->remove("${strBackupClusterPath}/${strBackup}", {bRecurse => true});
}
}
# If archive retention is still undefined, then ignore archiving
if (!defined($iArchiveRetention))
{
&log(INFO, "option '" . cfgOptionName(CFGOPT_REPO_RETENTION_ARCHIVE) . "' is not set - archive logs will not be expired");
}
else
{
my @stryGlobalBackupRetention;
# Determine which backup type to use for archive retention (full, differential, incremental) and get a list of the
# remaining non-expired backups based on the type.
if ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_FULL)
{
@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true), 'reverse');
}
elsif ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_DIFF)
{
@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true, true), 'reverse');
}
elsif ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_INCR)
{
@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true, true, true), 'reverse');
}
# If no backups were found then preserve current archive logs - too soon to expire them
my $iBackupTotal = scalar @stryGlobalBackupRetention;
if ($iBackupTotal > 0)
{
my $oArchiveInfo = new pgBackRest::Archive::Info($oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), true);
my @stryListArchiveDisk = sort {((split('-', $a))[1] + 0) cmp ((split('-', $b))[1] + 0)} $oStorageRepo->list(
STORAGE_REPO_ARCHIVE, {strExpression => REGEX_ARCHIVE_DIR_DB_VERSION, bIgnoreMissing => true});
# Make sure the current database versions match between the two files
if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
($oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION)))) ||
!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef,
($oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID)))))
{
confess &log(ERROR, "archive and backup database versions do not match\n" .
"HINT: has a stanza-upgrade been performed?", ERROR_FILE_INVALID);
}
# Get the list of backups that are part of archive retention
my @stryTmp = @stryGlobalBackupRetention;
my @stryGlobalBackupArchiveRetention = splice(@stryTmp, 0, $iArchiveRetention);
# For each archiveId, remove WAL that are not part of retention
foreach my $strArchiveId (@stryListArchiveDisk)
{
# From the global list of backups to retain, create a list of backups, oldest to newest, associated with this
# archiveId (e.g. 9.4-1)
my @stryLocalBackupRetention = $oBackupInfo->listByArchiveId($strArchiveId,
$oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \@stryGlobalBackupRetention, 'reverse');
# If no backup to retain was found
if (!@stryLocalBackupRetention)
{
# Get the backup db-id corresponding to this archiveId
my $iDbHistoryId = $oBackupInfo->backupArchiveDbHistoryId(
$strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE));
# If this is not the current database, then delete the archive directory else do nothing since the current
# DB archive directory must not be deleted
if (!defined($iDbHistoryId) || !$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef,
$iDbHistoryId))
{
my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}");
$oStorageRepo->remove($strFullPath, {bRecurse => true});
&log(INFO, "remove archive path: ${strFullPath}");
}
# Continue to next directory
next;
}
my @stryLocalBackupArchiveRentention;
# If the archive retention is less than or equal to the number of all backups, then perform selective expiration
if (@stryGlobalBackupArchiveRetention && $iArchiveRetention <= scalar @stryGlobalBackupRetention)
{
# From the full list of backups in archive retention, find the intersection of local backups to retain
foreach my $strGlobalBackupArchiveRetention (@stryGlobalBackupArchiveRetention)
{
foreach my $strLocalBackupRetention (@stryLocalBackupRetention)
{
if ($strLocalBackupRetention eq $strGlobalBackupArchiveRetention)
{
unshift(@stryLocalBackupArchiveRentention, $strLocalBackupRetention);
}
}
}
}
# Else if there are not enough backups yet globally to start archive expiration then set the archive retention
# to the oldest backup so anything prior to that will be removed as it is not needed but everything else is
# This is incase there are old archives left around so that they don't stay around forever
else
{
if ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_FULL && scalar @stryLocalBackupRetention > 0)
{
&log(INFO, "full backup total < ${iArchiveRetention} - using oldest full backup for ${strArchiveId} " .
"archive retention");
$stryLocalBackupArchiveRentention[0] = $stryLocalBackupRetention[0];
}
}
# If no local backups were found as part of retention then set the backup archive retention to the newest backup
# so that the database is fully recoverable (can be recovered from the last backup through pitr)
if (!@stryLocalBackupArchiveRentention)
{
$stryLocalBackupArchiveRentention[0] = $stryLocalBackupRetention[-1];
}
my $strArchiveRetentionBackup = $stryLocalBackupArchiveRentention[0];
# If a backup has been selected for retention then continue
if (defined($strArchiveRetentionBackup))
{
my $bRemove;
# Only expire if the selected backup has archive info - backups performed with --no-online will
# not have archive info and cannot be used for expiration.
if ($oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT,
$strArchiveRetentionBackup, INFO_BACKUP_KEY_ARCHIVE_START))
{
# Get archive ranges to preserve. Because archive retention can be less than total retention it is
# important to preserve archive that is required to make the older backups consistent even though they
# cannot be played any further forward with PITR.
my $strArchiveExpireMax;
my @oyArchiveRange;
my @stryBackupList = $oBackupInfo->list();
# With the full list of backups, loop through only those associated with this archiveId
foreach my $strBackup (
$oBackupInfo->listByArchiveId(
$strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \@stryBackupList))
{
if ($strBackup le $strArchiveRetentionBackup &&
$oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_ARCHIVE_START))
{
my $oArchiveRange = {};
$$oArchiveRange{start} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT,
$strBackup, INFO_BACKUP_KEY_ARCHIVE_START);
if ($strBackup ne $strArchiveRetentionBackup)
{
$$oArchiveRange{stop} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT,
$strBackup, INFO_BACKUP_KEY_ARCHIVE_STOP);
}
else
{
$strArchiveExpireMax = $$oArchiveRange{start};
}
&log(DETAIL, "archive retention on backup ${strBackup}, archiveId = ${strArchiveId}, " .
"start = $$oArchiveRange{start}" .
(defined($$oArchiveRange{stop}) ? ", stop = $$oArchiveRange{stop}" : ''));
push(@oyArchiveRange, $oArchiveRange);
}
}
# Get all major archive paths (timeline and first 32 bits of LSN)
foreach my $strPath ($oStorageRepo->list(
STORAGE_REPO_ARCHIVE . "/${strArchiveId}", {strExpression => REGEX_ARCHIVE_DIR_WAL}))
{
logDebugMisc($strOperation, "found major WAL path: ${strPath}");
$bRemove = true;
# Keep the path if it falls in the range of any backup in retention
foreach my $oArchiveRange (@oyArchiveRange)
{
if ($strPath ge substr($$oArchiveRange{start}, 0, 16) &&
(!defined($$oArchiveRange{stop}) || $strPath le substr($$oArchiveRange{stop}, 0, 16)))
{
$bRemove = false;
last;
}
}
# Remove the entire directory if all archive is expired
if ($bRemove)
{
my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}") . "/${strPath}";
$oStorageRepo->remove($strFullPath, {bRecurse => true});
# Log expire info
logDebugMisc($strOperation, "remove major WAL path: ${strFullPath}");
$self->logExpire($strArchiveId, $strPath);
}
# Else delete individual files instead if the major path is less than or equal to the most recent
# retention backup. This optimization prevents scanning though major paths that could not possibly
# have anything to expire.
elsif ($strPath le substr($strArchiveExpireMax, 0, 16))
{
# Look for files in the archive directory
foreach my $strSubPath ($oStorageRepo->list(
STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strPath}", {strExpression => "^[0-F]{24}.*\$"}))
{
$bRemove = true;
# Determine if the individual archive log is used in a backup
foreach my $oArchiveRange (@oyArchiveRange)
{
if (substr($strSubPath, 0, 24) ge $$oArchiveRange{start} &&
(!defined($$oArchiveRange{stop}) || substr($strSubPath, 0, 24) le $$oArchiveRange{stop}))
{
$bRemove = false;
last;
}
}
# Remove archive log if it is not used in a backup
if ($bRemove)
{
$oStorageRepo->remove(STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strSubPath}");
logDebugMisc($strOperation, "remove WAL segment: ${strArchiveId}/${strSubPath}");
# Log expire info
$self->logExpire($strArchiveId, substr($strSubPath, 0, 24));
}
else
{
# Log that the file was not expired
$self->logExpire($strArchiveId);
}
}
}
}
# Log if no archive was expired
if ($self->{iArchiveExpireTotal} == 0)
{
&log(DETAIL, "no archive to remove, archiveId = ${strArchiveId}");
}
}
}
}
}
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
1;

8
lib/pgBackRest/Main.pm

@ -13,11 +13,13 @@ $SIG{__DIE__} = sub {Carp::confess @_};
use File::Basename qw(dirname);
use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
@ -245,11 +247,7 @@ sub main
# --------------------------------------------------------------------------------------------------------------
elsif (cfgCommandTest(CFGCMD_EXPIRE))
{
# Load module dynamically
require pgBackRest::Expire;
pgBackRest::Expire->import();
new pgBackRest::Expire()->process();
new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP));
}
}
}

7
src/Makefile.in

@ -50,6 +50,8 @@ SRCS = \
command/archive/push/file.c \
command/archive/push/protocol.c \
command/archive/push/push.c \
command/backup/common.c \
command/expire/expire.c \
command/help/help.c \
command/info/info.c \
command/command.c \
@ -223,6 +225,9 @@ command/command.o: command/command.c build.auto.h common/assert.h common/debug.h
command/control/control.o: command/control/control.c build.auto.h command/control/control.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/control/control.c -o command/control/control.o
command/expire/expire.o: command/expire/expire.c build.auto.h command/archive/common.h command/backup/common.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoManifest.h info/infoPg.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/expire/expire.c -o command/expire/expire.o
command/help/help.o: command/help/help.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/help/help.c -o command/help/help.o
@ -421,7 +426,7 @@ info/infoManifest.o: info/infoManifest.c build.auto.h common/error.auto.h common
info/infoPg.o: info/infoPg.c build.auto.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h info/info.h info/infoPg.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoPg.c -o info/infoPg.o
main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/command.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h version.h
main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/command.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c main.c -o main.o
perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h

671
src/command/expire/expire.c Normal file

@ -0,0 +1,671 @@
/***********************************************************************************************************************************
Expire Command
***********************************************************************************************************************************/
#include "build.auto.h"
#include "command/archive/common.h"
#include "command/backup/common.h"
#include "common/type/list.h"
#include "common/debug.h"
#include "common/regExp.h"
#include "config/config.h"
#include "info/info.h"
#include "info/infoArchive.h"
#include "info/infoBackup.h"
#include "info/infoManifest.h"
#include "storage/helper.h"
#include <stdlib.h>
/***********************************************************************************************************************************
Helper functions and structures
***********************************************************************************************************************************/
typedef struct ArchiveExpired
{
uint64_t total;
String *start;
String *stop;
} ArchiveExpired;
static int
archiveIdAscComparator(const void *item1, const void *item2)
{
StringList *archiveSort1 = strLstNewSplitZ(*(String **)item1, "-");
StringList *archiveSort2 = strLstNewSplitZ(*(String **)item2, "-");
int int1 = atoi(strPtr(strLstGet(archiveSort1, 1)));
int int2 = atoi(strPtr(strLstGet(archiveSort2, 1)));
return (int1 - int2);
}
static int
archiveIdDescComparator(const void *item1, const void *item2)
{
StringList *archiveSort1 = strLstNewSplitZ(*(String **)item1, "-");
StringList *archiveSort2 = strLstNewSplitZ(*(String **)item2, "-");
int int1 = atoi(strPtr(strLstGet(archiveSort1, 1)));
int int2 = atoi(strPtr(strLstGet(archiveSort2, 1)));
return (int2 - int1);
}
static StringList *
sortArchiveId(StringList *sortString, SortOrder sortOrder)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING_LIST, sortString);
FUNCTION_LOG_PARAM(ENUM, sortOrder);
FUNCTION_LOG_END();
ASSERT(sortString != NULL);
switch (sortOrder)
{
case sortOrderAsc:
{
lstSort((List *)sortString, archiveIdAscComparator);
break;
}
case sortOrderDesc:
{
lstSort((List *)sortString, archiveIdDescComparator);
break;
}
}
FUNCTION_LOG_RETURN(STRING_LIST, sortString);
}
typedef struct ArchiveRange
{
const String *start;
const String *stop;
} ArchiveRange;
/***********************************************************************************************************************************
Common function for expiring any backup
***********************************************************************************************************************************/
static void
expireBackup(InfoBackup *infoBackup, String *removeBackupLabel, String *backupExpired)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_PARAM(STRING, removeBackupLabel);
FUNCTION_LOG_PARAM(STRING, backupExpired);
FUNCTION_LOG_END();
ASSERT(infoBackup != NULL);
ASSERT(removeBackupLabel != NULL);
ASSERT(backupExpired != NULL);
storageRemoveNP(
storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" INFO_MANIFEST_FILE, strPtr(removeBackupLabel)));
storageRemoveNP(
storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" INFO_MANIFEST_FILE INFO_COPY_EXT, strPtr(removeBackupLabel)));
// Remove the backup from the info file
infoBackupDataDelete(infoBackup, removeBackupLabel);
if (strSize(backupExpired) == 0)
strCat(backupExpired, strPtr(removeBackupLabel));
else
strCatFmt(backupExpired, ", %s", strPtr(removeBackupLabel));
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Expire differential backups
***********************************************************************************************************************************/
static unsigned int
expireDiffBackup(InfoBackup *infoBackup)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_END();
ASSERT(infoBackup != NULL);
unsigned int result = 0;
MEM_CONTEXT_TEMP_BEGIN()
{
unsigned int differentialRetention = cfgOptionTest(cfgOptRepoRetentionDiff) ? cfgOptionUInt(cfgOptRepoRetentionDiff) : 0;
// Find all the expired differential backups
if (differentialRetention > 0)
{
// Get a list of full and differential backups. Full are considered differential for the purpose of retention.
// Example: F1, D1, D2, F2, repo-retention-diff=2, then F1,D2,F2 will be retained, not D2 and D1 as might be expected.
StringList *currentBackupList = infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true, .differential = true));
// If there are more backups than the number to retain, then expire the oldest ones
if (strLstSize(currentBackupList) > differentialRetention)
{
for (unsigned int diffIdx = 0; diffIdx < strLstSize(currentBackupList) - differentialRetention; diffIdx++)
{
// Skip if this is a full backup. Full backups only count as differential when deciding which differential
// backups to expire.
if (regExpMatchOne(backupRegExpP(.full = true), strLstGet(currentBackupList, diffIdx)))
continue;
// Get a list of all differential and incremental backups
StringList *removeList = infoBackupDataLabelList(
infoBackup, backupRegExpP(.differential = true, .incremental = true));
// Initialize the log message
String *backupExpired = strNew("");
// Remove the manifest files in each directory and remove the backup from the current section of backup.info
for (unsigned int rmvIdx = 0; rmvIdx < strLstSize(removeList); rmvIdx++)
{
// Remove all differential and incremental backups before the oldest valid differential
// (removeBackupLabel < oldest valid differential)
String *removeBackupLabel = strLstGet(removeList, rmvIdx);
if (strCmp(removeBackupLabel, strLstGet(currentBackupList, diffIdx + 1)) < 0)
{
expireBackup(infoBackup, removeBackupLabel, backupExpired);
result++;
}
}
// If the message contains a comma, then prepend "set:"
LOG_INFO("expire diff backup %s%s", (strChr(backupExpired, ',') != -1 ? "set: " : ""), strPtr(backupExpired));
}
}
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(UINT, result);
}
/***********************************************************************************************************************************
Expire full backups
***********************************************************************************************************************************/
static unsigned int
expireFullBackup(InfoBackup *infoBackup)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_END();
ASSERT(infoBackup != NULL);
unsigned int result = 0;
MEM_CONTEXT_TEMP_BEGIN()
{
unsigned int fullRetention = cfgOptionTest(cfgOptRepoRetentionFull) ? cfgOptionUInt(cfgOptRepoRetentionFull) : 0;
// Find all the expired full backups
if (fullRetention > 0)
{
// Get list of current full backups (default order is oldest to newest)
StringList *currentBackupList = infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true));
// If there are more full backups then the number to retain, then expire the oldest ones
if (strLstSize(currentBackupList) > fullRetention)
{
// Expire all backups that depend on the full backup
for (unsigned int fullIdx = 0; fullIdx < strLstSize(currentBackupList) - fullRetention; fullIdx++)
{
// The list of backups to remove includes the full backup and the default sort order will put it first
StringList *removeList = infoBackupDataLabelList(
infoBackup, strNewFmt("^%s.*", strPtr(strLstGet(currentBackupList, fullIdx))));
// Initialize the log message
String *backupExpired = strNew("");
// Remove the manifest files in each directory and remove the backup from the current section of backup.info
for (unsigned int rmvIdx = 0; rmvIdx < strLstSize(removeList); rmvIdx++)
{
String *removeBackupLabel = strLstGet(removeList, rmvIdx);
expireBackup(infoBackup, removeBackupLabel, backupExpired);
result++;
}
// If the message contains a comma, then prepend "set:"
LOG_INFO("expire full backup %s%s", (strChr(backupExpired, ',') != -1 ? "set: " : ""), strPtr(backupExpired));
}
}
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(UINT, result);
}
/***********************************************************************************************************************************
Log detailed information about archive logs removed
***********************************************************************************************************************************/
static void
logExpire(ArchiveExpired *archiveExpire, String *archiveId)
{
if (archiveExpire->start != NULL)
{
// Force out any remaining message
LOG_DETAIL(
"remove archive: archiveId = %s, start = %s, stop = %s", strPtr(archiveId), strPtr(archiveExpire->start),
strPtr(archiveExpire->stop));
archiveExpire->start = NULL;
}
}
/***********************************************************************************************************************************
Process archive retention
***********************************************************************************************************************************/
static void
removeExpiredArchive(InfoBackup *infoBackup)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_END();
ASSERT(infoBackup != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the retention options. repo-archive-retention-type always has a value as it defaults to "full"
const String *archiveRetentionType = cfgOptionStr(cfgOptRepoRetentionArchiveType);
unsigned int archiveRetention = cfgOptionTest(cfgOptRepoRetentionArchive) ? cfgOptionUInt(cfgOptRepoRetentionArchive) : 0;
// If archive retention is undefined, then ignore archiving. The user does not have to set this - it will be defaulted in
// cfgLoadUpdateOption based on certain rules.
if (archiveRetention == 0)
{
LOG_INFO("option '%s' is not set - archive logs will not be expired", cfgOptionName(cfgOptRepoRetentionArchive));
}
else
{
// Determine which backup type to use for archive retention (full, differential, incremental) and get a list of the
// remaining non-expired backups, from newest to oldest, based on the type.
StringList *globalBackupRetentionList = NULL;
if (strCmp(archiveRetentionType, STRDEF(CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL)) == 0)
{
globalBackupRetentionList = strLstSort(
infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true)), sortOrderDesc);
}
else if (strCmp(archiveRetentionType, STRDEF(CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_DIFF)) == 0)
{
globalBackupRetentionList = strLstSort(
infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true, .differential = true)), sortOrderDesc);
}
else
{ // Incrementals can depend on Full or Diff so get a list of all incrementals
globalBackupRetentionList = strLstSort(
infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true, .differential = true, .incremental = true)),
sortOrderDesc);
}
// Expire archives. If no backups were found then preserve current archive logs - too soon to expire them.
if (strLstSize(globalBackupRetentionList) > 0)
{
// Attempt to load the archive info file
InfoArchive *infoArchive = infoArchiveNewLoad(
storageRepo(), STRDEF(STORAGE_REPO_ARCHIVE "/" INFO_ARCHIVE_FILE),
cipherType(cfgOptionStr(cfgOptRepoCipherType)), cfgOptionStr(cfgOptRepoCipherPass));
InfoPg *infoArchivePgData = infoArchivePg(infoArchive);
// Get a list of archive directories (e.g. 9.4-1, 10-2, etc) sorted by the db-id (number after the dash).
StringList *listArchiveDisk = sortArchiveId(
storageListP(storageRepo(), STRDEF(STORAGE_REPO_ARCHIVE), .expression = STRDEF(REGEX_ARCHIVE_DIR_DB_VERSION)),
sortOrderAsc);
StringList *globalBackupArchiveRetentionList = strLstNew();
// globalBackupRetentionList is ordered newest to oldest backup, so create globalBackupArchiveRetentionList of the
// newest backups whose archives will be retained
for (unsigned int idx = 0;
idx < (archiveRetention < strLstSize(globalBackupRetentionList) ?
archiveRetention : strLstSize(globalBackupRetentionList));
idx++)
{
strLstAdd(globalBackupArchiveRetentionList, strLstGet(globalBackupRetentionList, idx));
}
// Loop through the archive.info history from oldest to newest and if there is a corresponding directory on disk
// then remove WAL that are not part of retention
for (unsigned int pgIdx = infoPgDataTotal(infoArchivePgData) - 1; (int)pgIdx >= 0; pgIdx--)
{
String *archiveId = infoPgArchiveId(infoArchivePgData, pgIdx);
StringList *localBackupRetentionList = strLstNew();
// Initialize the expired archive information for this archive ID
ArchiveExpired archiveExpire = {.total = 0, .start = NULL, .stop = NULL};
for (unsigned int archiveIdx = 0; archiveIdx < strLstSize(listArchiveDisk); archiveIdx++)
{
// Is there an archive directory for this archvieId?
if (strCmp(archiveId, strLstGet(listArchiveDisk, archiveIdx)) != 0)
continue;
StringList *archiveSplit = strLstNewSplitZ(archiveId, "-");
unsigned int archivePgId = cvtZToUInt(strPtr(strLstGet(archiveSplit, 1)));
// From the global list of backups to retain, create a list of backups, oldest to newest, associated with
// this archiveId (e.g. 9.4-1), e.g. If globalBackupRetention has 4F, 3F, 2F, 1F then
// localBackupRetentionList will have 1F, 2F, 3F, 4F (assuming they all have same history id)
for (unsigned int retentionIdx = strLstSize(globalBackupRetentionList) - 1;
(int)retentionIdx >=0; retentionIdx--)
{
for (unsigned int backupIdx = 0; backupIdx < infoBackupDataTotal(infoBackup); backupIdx++)
{
InfoBackupData backupData = infoBackupData(infoBackup, backupIdx);
if (strCmp(backupData.backupLabel, strLstGet(globalBackupRetentionList, retentionIdx)) == 0 &&
backupData.backupPgId == archivePgId)
{
strLstAdd(localBackupRetentionList, backupData.backupLabel);
}
}
}
// If no backup to retain was found
if (strLstSize(localBackupRetentionList) == 0)
{
// If this is not the current database, then delete the archive directory else do nothing since the
// current DB archive directory must not be deleted
InfoPgData currentPg = infoPgDataCurrent(infoArchivePgData);
if (currentPg.id != archivePgId)
{
String *fullPath = storagePath(
storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strPtr(archiveId)));
storagePathRemoveP(storageRepoWrite(), fullPath, .recurse = true);
LOG_INFO("remove archive path: %s", strPtr(fullPath));
}
// Continue to next directory
break;
}
// If we get here, then a local backup was found for retention
StringList *localBackupArchiveRententionList = strLstNew();
// If the archive retention is less than or equal to the number of all backups, then perform selective
// expiration
if (archiveRetention <= strLstSize(globalBackupRetentionList))
{
// From the full list of backups in archive retention, find the intersection of local backups to retain
// from oldest to newest
for (unsigned int globalIdx = strLstSize(globalBackupArchiveRetentionList) - 1;
(int)globalIdx >= 0; globalIdx--)
{
for (unsigned int localIdx = 0; localIdx < strLstSize(localBackupRetentionList); localIdx++)
{
if (strCmp(
strLstGet(globalBackupArchiveRetentionList, globalIdx),
strLstGet(localBackupRetentionList, localIdx)) == 0)
{
strLstAdd(localBackupArchiveRententionList, strLstGet(localBackupRetentionList, localIdx));
}
}
}
}
// Else if there are not enough backups yet globally to start archive expiration then set the archive
// retention to the oldest backup so anything prior to that will be removed as it is not needed but
// everything else is. This is incase there are old archives left around so that they don't stay around
// forever.
else
{
LOG_INFO(
"full backup total < %u - using oldest full backup for %s archive retention", archiveRetention,
strPtr(archiveId));
strLstAdd(localBackupArchiveRententionList, strLstGet(localBackupRetentionList, 0));
}
// If no local backups were found as part of retention then set the backup archive retention to the newest
// backup so that the database is fully recoverable (can be recovered from the last backup through pitr)
if (strLstSize(localBackupArchiveRententionList) == 0)
{
strLstAdd(
localBackupArchiveRententionList,
strLstGet(localBackupRetentionList, strLstSize(localBackupRetentionList) - 1));
}
// Get the data for the backup selected for retention and all backups associated with this archive id
List *archiveIdBackupList = lstNew(sizeof(InfoBackupData));
InfoBackupData archiveRetentionBackup = {0};
for (unsigned int infoBackupIdx = 0; infoBackupIdx < infoBackupDataTotal(infoBackup); infoBackupIdx++)
{
InfoBackupData archiveIdBackup = infoBackupData(infoBackup, infoBackupIdx);
// If this is the backup selected for retention, store its data
if (strCmp(archiveIdBackup.backupLabel, strLstGet(localBackupArchiveRententionList, 0)) == 0)
archiveRetentionBackup = infoBackupData(infoBackup, infoBackupIdx);
// If this is a backup associated with this archive Id, then add it to the list to check
if (archiveIdBackup.backupPgId == archivePgId)
lstAdd(archiveIdBackupList, &archiveIdBackup);
}
// Only expire if the selected backup has archive data - backups performed with --no-online will
// not have archive data and cannot be used for expiration.
bool removeArchive = false;
if (archiveRetentionBackup.backupArchiveStart != NULL)
{
// Get archive ranges to preserve. Because archive retention can be less than total retention it is
// important to preserve archive that is required to make the older backups consistent even though they
// cannot be played any further forward with PITR.
String *archiveExpireMax = NULL;
List *archiveRangeList = lstNew(sizeof(ArchiveRange));
// From the full list of backups, loop through those associated with this archiveId
for (unsigned int backupListIdx = 0; backupListIdx < lstSize(archiveIdBackupList); backupListIdx++)
{
InfoBackupData *backupData = lstGet(archiveIdBackupList, backupListIdx);
// If the backup is earlier than or the same as the retention backup and the backup has an
// archive start
if (strCmp(backupData->backupLabel, archiveRetentionBackup.backupLabel) <= 0 &&
backupData->backupArchiveStart != NULL)
{
ArchiveRange archiveRange =
{
.start = strDup(backupData->backupArchiveStart),
.stop = NULL,
};
// If this is not the retention backup, then set the stop, otherwise set the expire max to
// the archive start of the archive to retain
if (strCmp(backupData->backupLabel, archiveRetentionBackup.backupLabel) != 0)
archiveRange.stop = strDup(backupData->backupArchiveStop);
else
archiveExpireMax = strDup(archiveRange.start);
LOG_DETAIL(
"archive retention on backup %s, archiveId = %s, start = %s%s",
strPtr(backupData->backupLabel), strPtr(archiveId), strPtr(archiveRange.start),
archiveRange.stop != NULL ?
strPtr(strNewFmt(", stop = %s", strPtr(archiveRange.stop))) : "");
// Add the archive range to the list
lstAdd(archiveRangeList, &archiveRange);
}
}
// Get all major archive paths (timeline and first 32 bits of LSN)
StringList *walPathList =
strLstSort(
storageListP(
storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strPtr(archiveId)),
.expression = STRDEF(WAL_SEGMENT_DIR_REGEXP)),
sortOrderAsc);
for (unsigned int walIdx = 0; walIdx < strLstSize(walPathList); walIdx++)
{
String *walPath = strLstGet(walPathList, walIdx);
removeArchive = true;
// Keep the path if it falls in the range of any backup in retention
for (unsigned int rangeIdx = 0; rangeIdx < lstSize(archiveRangeList); rangeIdx++)
{
ArchiveRange *archiveRange = lstGet(archiveRangeList, rangeIdx);
if (strCmp(walPath, strSubN(archiveRange->start, 0, 16)) >= 0 &&
(archiveRange->stop == NULL || strCmp(walPath, strSubN(archiveRange->stop, 0, 16)) <= 0))
{
removeArchive = false;
break;
}
}
// Remove the entire directory if all archive is expired
if (removeArchive)
{
storagePathRemoveP(
storageRepoWrite(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId),
strPtr(walPath)), .recurse = true);
archiveExpire.total++;
archiveExpire.start = strDup(walPath);
archiveExpire.stop = strDup(walPath);
}
// Else delete individual files instead if the major path is less than or equal to the most recent
// retention backup. This optimization prevents scanning though major paths that could not possibly
// have anything to expire.
else if (strCmp(walPath, strSubN(archiveExpireMax, 0, 16)) <= 0)
{
// Look for files in the archive directory
StringList *walSubPathList =
strLstSort(
storageListP(
storageRepo(),
strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(walPath)),
.expression = STRDEF("^[0-F]{24}.*$")),
sortOrderAsc);
for (unsigned int subIdx = 0; subIdx < strLstSize(walSubPathList); subIdx++)
{
removeArchive = true;
String *walSubPath = strLstGet(walSubPathList, subIdx);
// Determine if the individual archive log is used in a backup
for (unsigned int rangeIdx = 0; rangeIdx < lstSize(archiveRangeList); rangeIdx++)
{
ArchiveRange *archiveRange = lstGet(archiveRangeList, rangeIdx);
if (strCmp(strSubN(walSubPath, 0, 24), archiveRange->start) >= 0 &&
(archiveRange->stop == NULL ||
strCmp(strSubN(walSubPath, 0, 24), archiveRange->stop) <= 0))
{
removeArchive = false;
break;
}
}
// Remove archive log if it is not used in a backup
if (removeArchive)
{
storageRemoveNP(
storageRepoWrite(),
strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s/%s",
strPtr(archiveId), strPtr(walPath), strPtr(walSubPath)));
// Track that this archive was removed
archiveExpire.total++;
archiveExpire.stop = strDup(strSubN(walSubPath, 0, 24));
if (archiveExpire.start == NULL)
archiveExpire.start = strDup(strSubN(walSubPath, 0, 24));
}
else
logExpire(&archiveExpire, archiveId);
}
}
}
// Log if no archive was expired
if (archiveExpire.total == 0)
{
LOG_DETAIL("no archive to remove, archiveId = %s", strPtr(archiveId));
}
// Log if there is more to log
else
logExpire(&archiveExpire, archiveId);
}
}
}
}
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Remove expired backups from repo
***********************************************************************************************************************************/
static void
removeExpiredBackup(InfoBackup *infoBackup)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_END();
ASSERT(infoBackup != NULL);
// Get all the current backups in backup.info
StringList *currentBackupList = strLstSort(infoBackupDataLabelList(infoBackup, NULL), sortOrderDesc);
StringList *backupList = strLstSort(
storageListP(
storageRepo(), STRDEF(STORAGE_REPO_BACKUP), .expression = backupRegExpP(.full = true, .differential = true,
.incremental = true)),
sortOrderDesc);
// Remove non-current backups from disk
for (unsigned int backupIdx = 0; backupIdx < strLstSize(backupList); backupIdx++)
{
if (!strLstExists(currentBackupList, strLstGet(backupList, backupIdx)))
{
LOG_INFO("remove expired backup %s", strPtr(strLstGet(backupList, backupIdx)));
storagePathRemoveP(
storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s", strPtr(strLstGet(backupList, backupIdx))),
.recurse = true);
}
}
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Expire backups and archives
***********************************************************************************************************************************/
void
cmdExpire(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the repo storage in case it is remote and encryption settings need to be pulled down
storageRepo();
// Load the backup.info
InfoBackup *infoBackup = infoBackupNewLoad(
storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/" INFO_BACKUP_FILE), cipherType(cfgOptionStr(cfgOptRepoCipherType)),
cfgOptionStr(cfgOptRepoCipherPass));
expireFullBackup(infoBackup);
expireDiffBackup(infoBackup);
infoBackupSave(infoBackup, storageRepoWrite(), STRDEF(STORAGE_REPO_BACKUP "/" INFO_BACKUP_FILE),
cipherType(cfgOptionStr(cfgOptRepoCipherType)), cfgOptionStr(cfgOptRepoCipherPass));
removeExpiredBackup(infoBackup);
removeExpiredArchive(infoBackup);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}

12
src/command/expire/expire.h Normal file

@ -0,0 +1,12 @@
/***********************************************************************************************************************************
Expire Command
***********************************************************************************************************************************/
#ifndef COMMAND_EXPIRE_EXPIRE_H
#define COMMAND_EXPIRE_EXPIRE_H
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void cmdExpire(void);
#endif

3
src/main.c

@ -10,6 +10,7 @@ Main
#include "command/archive/get/get.h"
#include "command/archive/push/push.h"
#include "command/command.h"
#include "command/expire/expire.h"
#include "command/help/help.h"
#include "command/info/info.h"
#include "command/local/local.h"
@ -115,6 +116,7 @@ main(int argListSize, const char *argList[])
// Run expire
perlExec();
cmdExpire();
break;
}
@ -132,6 +134,7 @@ main(int argListSize, const char *argList[])
case cfgCmdExpire:
{
perlExec();
cmdExpire();
break;
}

409
src/perl/embed.auto.c

@ -8814,407 +8814,6 @@ static const EmbeddedModule embeddedModule[] =
"\n"
"1;\n"
},
{
.name = "pgBackRest/Expire.pm",
.data =
"\n\n\n"
"package pgBackRest::Expire;\n"
"\n"
"use strict;\n"
"use warnings FATAL => qw(all);\n"
"use Carp qw(confess);\n"
"\n"
"use Exporter qw(import);\n"
"use File::Basename qw(dirname);\n"
"use Scalar::Util qw(looks_like_number);\n"
"\n"
"use pgBackRest::Archive::Common;\n"
"use pgBackRest::Archive::Info;\n"
"use pgBackRest::Common::Exception;\n"
"use pgBackRest::Common::Ini;\n"
"use pgBackRest::Common::Log;\n"
"use pgBackRest::Backup::Common;\n"
"use pgBackRest::Backup::Info;\n"
"use pgBackRest::Config::Config;\n"
"use pgBackRest::InfoCommon;\n"
"use pgBackRest::Manifest;\n"
"use pgBackRest::Protocol::Helper;\n"
"use pgBackRest::Protocol::Storage::Helper;\n"
"\n\n\n\n"
"sub new\n"
"{\n"
"my $class = shift;\n"
"\n\n"
"my $self = {};\n"
"bless $self, $class;\n"
"\n\n"
"my ($strOperation) = logDebugParam(__PACKAGE__ . '->new');\n"
"\n\n"
"$self->{iArchiveExpireTotal} = 0;\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'self', value => $self}\n"
");\n"
"}\n"
"\n\n\n\n\n\n"
"sub logExpire\n"
"{\n"
"my $self = shift;\n"
"my $strArchiveId = shift;\n"
"my $strArchiveFile = shift;\n"
"\n"
"if (defined($strArchiveFile))\n"
"{\n"
"if (!defined($self->{strArchiveExpireStart}))\n"
"{\n"
"$self->{strArchiveExpireStart} = $strArchiveFile;\n"
"$self->{strArchiveExpireStop} = $strArchiveFile;\n"
"}\n"
"else\n"
"{\n"
"$self->{strArchiveExpireStop} = $strArchiveFile;\n"
"}\n"
"\n"
"$self->{iArchiveExpireTotal}++;\n"
"}\n"
"else\n"
"{\n"
"if (defined($self->{strArchiveExpireStart}))\n"
"{\n"
"&log(DETAIL, \"remove archive: archiveId = ${strArchiveId}, start = \" . substr($self->{strArchiveExpireStart}, 0, 24) .\n"
"\", stop = \" . substr($self->{strArchiveExpireStop}, 0, 24));\n"
"}\n"
"\n"
"undef($self->{strArchiveExpireStart});\n"
"}\n"
"}\n"
"\n\n\n\n\n\n\n"
"sub process\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my ($strOperation) = logDebugParam(__PACKAGE__ . '->process');\n"
"\n"
"my @stryPath;\n"
"\n"
"my $oStorageRepo = storageRepo();\n"
"my $strBackupClusterPath = $oStorageRepo->pathGet(STORAGE_REPO_BACKUP);\n"
"my $iFullRetention = cfgOption(CFGOPT_REPO_RETENTION_FULL, false);\n"
"my $iDifferentialRetention = cfgOption(CFGOPT_REPO_RETENTION_DIFF, false);\n"
"my $strArchiveRetentionType = cfgOption(CFGOPT_REPO_RETENTION_ARCHIVE_TYPE, false);\n"
"my $iArchiveRetention = cfgOption(CFGOPT_REPO_RETENTION_ARCHIVE, false);\n"
"\n\n"
"my $oBackupInfo = new pgBackRest::Backup::Info($oStorageRepo->pathGet(STORAGE_REPO_BACKUP));\n"
"\n\n"
"if (defined($iFullRetention))\n"
"{\n"
"\n"
"if (!looks_like_number($iFullRetention) || $iFullRetention < 1)\n"
"{\n"
"confess &log(ERROR, cfgOptionName(CFGOPT_REPO_RETENTION_FULL) . ' must be a number >= 1');\n"
"}\n"
"\n"
"@stryPath = $oBackupInfo->list(backupRegExpGet(true));\n"
"\n"
"if (@stryPath > $iFullRetention)\n"
"{\n"
"\n"
"for (my $iFullIdx = 0; $iFullIdx < @stryPath - $iFullRetention; $iFullIdx++)\n"
"{\n"
"my @stryRemoveList;\n"
"\n"
"foreach my $strPath ($oBackupInfo->list('^' . $stryPath[$iFullIdx] . '.*'))\n"
"{\n"
"$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strPath}/\" . FILE_MANIFEST . INI_COPY_EXT);\n"
"$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strPath}/\" . FILE_MANIFEST);\n"
"$oBackupInfo->delete($strPath);\n"
"\n"
"if ($strPath ne $stryPath[$iFullIdx])\n"
"{\n"
"push(@stryRemoveList, $strPath);\n"
"}\n"
"}\n"
"\n"
"&log(INFO, 'expire full backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iFullIdx] .\n"
"(@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : ''));\n"
"}\n"
"}\n"
"}\n"
"\n\n"
"if (defined($iDifferentialRetention))\n"
"{\n"
"\n"
"if (!looks_like_number($iDifferentialRetention) || $iDifferentialRetention < 1)\n"
"{\n"
"confess &log(ERROR, cfgOptionName(CFGOPT_REPO_RETENTION_DIFF) . ' must be a number >= 1');\n"
"}\n"
"\n\n\n"
"@stryPath = $oBackupInfo->list(backupRegExpGet(true, true));\n"
"\n"
"if (@stryPath > $iDifferentialRetention)\n"
"{\n"
"for (my $iDiffIdx = 0; $iDiffIdx < @stryPath - $iDifferentialRetention; $iDiffIdx++)\n"
"{\n"
"\n\n"
"next if ($stryPath[$iDiffIdx] =~ backupRegExpGet(true));\n"
"\n\n"
"my @stryRemoveList;\n"
"\n"
"foreach my $strPath ($oBackupInfo->list(backupRegExpGet(false, true, true)))\n"
"{\n"
"logDebugMisc($strOperation, \"checking ${strPath} for differential expiration\");\n"
"\n\n"
"if ($strPath lt $stryPath[$iDiffIdx + 1])\n"
"{\n"
"$oStorageRepo->remove(STORAGE_REPO_BACKUP . \"/${strPath}\" . FILE_MANIFEST);\n"
"$oBackupInfo->delete($strPath);\n"
"\n"
"if ($strPath ne $stryPath[$iDiffIdx])\n"
"{\n"
"push(@stryRemoveList, $strPath);\n"
"}\n"
"}\n"
"}\n"
"\n"
"&log(INFO, 'expire diff backup ' . (@stryRemoveList > 0 ? 'set: ' : '') . $stryPath[$iDiffIdx] .\n"
"(@stryRemoveList > 0 ? ', ' . join(', ', @stryRemoveList) : ''));\n"
"}\n"
"}\n"
"}\n"
"\n"
"$oBackupInfo->save();\n"
"\n\n"
"foreach my $strBackup ($oStorageRepo->list(\n"
"STORAGE_REPO_BACKUP, {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))\n"
"{\n"
"if (!$oBackupInfo->current($strBackup))\n"
"{\n"
"&log(INFO, \"remove expired backup ${strBackup}\");\n"
"\n"
"$oStorageRepo->remove(\"${strBackupClusterPath}/${strBackup}\", {bRecurse => true});\n"
"}\n"
"}\n"
"\n\n"
"if (!defined($iArchiveRetention))\n"
"{\n"
"&log(INFO, \"option '\" . cfgOptionName(CFGOPT_REPO_RETENTION_ARCHIVE) . \"' is not set - archive logs will not be expired\");\n"
"}\n"
"else\n"
"{\n"
"my @stryGlobalBackupRetention;\n"
"\n\n\n"
"if ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_FULL)\n"
"{\n"
"@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true), 'reverse');\n"
"}\n"
"elsif ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_DIFF)\n"
"{\n"
"@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true, true), 'reverse');\n"
"}\n"
"elsif ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_INCR)\n"
"{\n"
"@stryGlobalBackupRetention = $oBackupInfo->list(backupRegExpGet(true, true, true), 'reverse');\n"
"}\n"
"\n\n"
"my $iBackupTotal = scalar @stryGlobalBackupRetention;\n"
"\n"
"if ($iBackupTotal > 0)\n"
"{\n"
"my $oArchiveInfo = new pgBackRest::Archive::Info($oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), true);\n"
"my @stryListArchiveDisk = sort {((split('-', $a))[1] + 0) cmp ((split('-', $b))[1] + 0)} $oStorageRepo->list(\n"
"STORAGE_REPO_ARCHIVE, {strExpression => REGEX_ARCHIVE_DIR_DB_VERSION, bIgnoreMissing => true});\n"
"\n\n"
"if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,\n"
"($oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION)))) ||\n"
"!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef,\n"
"($oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID)))))\n"
"{\n"
"confess &log(ERROR, \"archive and backup database versions do not match\\n\" .\n"
"\"HINT: has a stanza-upgrade been performed?\", ERROR_FILE_INVALID);\n"
"}\n"
"\n\n"
"my @stryTmp = @stryGlobalBackupRetention;\n"
"my @stryGlobalBackupArchiveRetention = splice(@stryTmp, 0, $iArchiveRetention);\n"
"\n\n"
"foreach my $strArchiveId (@stryListArchiveDisk)\n"
"{\n"
"\n\n"
"my @stryLocalBackupRetention = $oBackupInfo->listByArchiveId($strArchiveId,\n"
"$oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \\@stryGlobalBackupRetention, 'reverse');\n"
"\n\n"
"if (!@stryLocalBackupRetention)\n"
"{\n"
"\n"
"my $iDbHistoryId = $oBackupInfo->backupArchiveDbHistoryId(\n"
"$strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE));\n"
"\n\n\n"
"if (!defined($iDbHistoryId) || !$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef,\n"
"$iDbHistoryId))\n"
"{\n"
"my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . \"/${strArchiveId}\");\n"
"\n"
"$oStorageRepo->remove($strFullPath, {bRecurse => true});\n"
"\n"
"&log(INFO, \"remove archive path: ${strFullPath}\");\n"
"}\n"
"\n\n"
"next;\n"
"}\n"
"\n"
"my @stryLocalBackupArchiveRentention;\n"
"\n\n"
"if (@stryGlobalBackupArchiveRetention && $iArchiveRetention <= scalar @stryGlobalBackupRetention)\n"
"{\n"
"\n"
"foreach my $strGlobalBackupArchiveRetention (@stryGlobalBackupArchiveRetention)\n"
"{\n"
"foreach my $strLocalBackupRetention (@stryLocalBackupRetention)\n"
"{\n"
"if ($strLocalBackupRetention eq $strGlobalBackupArchiveRetention)\n"
"{\n"
"unshift(@stryLocalBackupArchiveRentention, $strLocalBackupRetention);\n"
"}\n"
"}\n"
"}\n"
"}\n"
"\n\n\n"
"else\n"
"{\n"
"if ($strArchiveRetentionType eq CFGOPTVAL_BACKUP_TYPE_FULL && scalar @stryLocalBackupRetention > 0)\n"
"{\n"
"&log(INFO, \"full backup total < ${iArchiveRetention} - using oldest full backup for ${strArchiveId} \" .\n"
"\"archive retention\");\n"
"$stryLocalBackupArchiveRentention[0] = $stryLocalBackupRetention[0];\n"
"}\n"
"}\n"
"\n\n\n"
"if (!@stryLocalBackupArchiveRentention)\n"
"{\n"
"$stryLocalBackupArchiveRentention[0] = $stryLocalBackupRetention[-1];\n"
"}\n"
"\n"
"my $strArchiveRetentionBackup = $stryLocalBackupArchiveRentention[0];\n"
"\n\n"
"if (defined($strArchiveRetentionBackup))\n"
"{\n"
"my $bRemove;\n"
"\n\n\n"
"if ($oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT,\n"
"$strArchiveRetentionBackup, INFO_BACKUP_KEY_ARCHIVE_START))\n"
"{\n"
"\n\n\n"
"my $strArchiveExpireMax;\n"
"my @oyArchiveRange;\n"
"my @stryBackupList = $oBackupInfo->list();\n"
"\n\n"
"foreach my $strBackup (\n"
"$oBackupInfo->listByArchiveId(\n"
"$strArchiveId, $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE), \\@stryBackupList))\n"
"{\n"
"if ($strBackup le $strArchiveRetentionBackup &&\n"
"$oBackupInfo->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_ARCHIVE_START))\n"
"{\n"
"my $oArchiveRange = {};\n"
"\n"
"$$oArchiveRange{start} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT,\n"
"$strBackup, INFO_BACKUP_KEY_ARCHIVE_START);\n"
"\n"
"if ($strBackup ne $strArchiveRetentionBackup)\n"
"{\n"
"$$oArchiveRange{stop} = $oBackupInfo->get(INFO_BACKUP_SECTION_BACKUP_CURRENT,\n"
"$strBackup, INFO_BACKUP_KEY_ARCHIVE_STOP);\n"
"}\n"
"else\n"
"{\n"
"$strArchiveExpireMax = $$oArchiveRange{start};\n"
"}\n"
"\n"
"&log(DETAIL, \"archive retention on backup ${strBackup}, archiveId = ${strArchiveId}, \" .\n"
"\"start = $$oArchiveRange{start}\" .\n"
"(defined($$oArchiveRange{stop}) ? \", stop = $$oArchiveRange{stop}\" : ''));\n"
"\n"
"push(@oyArchiveRange, $oArchiveRange);\n"
"}\n"
"}\n"
"\n\n"
"foreach my $strPath ($oStorageRepo->list(\n"
"STORAGE_REPO_ARCHIVE . \"/${strArchiveId}\", {strExpression => REGEX_ARCHIVE_DIR_WAL}))\n"
"{\n"
"logDebugMisc($strOperation, \"found major WAL path: ${strPath}\");\n"
"$bRemove = true;\n"
"\n\n"
"foreach my $oArchiveRange (@oyArchiveRange)\n"
"{\n"
"if ($strPath ge substr($$oArchiveRange{start}, 0, 16) &&\n"
"(!defined($$oArchiveRange{stop}) || $strPath le substr($$oArchiveRange{stop}, 0, 16)))\n"
"{\n"
"$bRemove = false;\n"
"last;\n"
"}\n"
"}\n"
"\n\n"
"if ($bRemove)\n"
"{\n"
"my $strFullPath = $oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE . \"/${strArchiveId}\") . \"/${strPath}\";\n"
"\n"
"$oStorageRepo->remove($strFullPath, {bRecurse => true});\n"
"\n\n"
"logDebugMisc($strOperation, \"remove major WAL path: ${strFullPath}\");\n"
"$self->logExpire($strArchiveId, $strPath);\n"
"}\n"
"\n\n\n"
"elsif ($strPath le substr($strArchiveExpireMax, 0, 16))\n"
"{\n"
"\n"
"foreach my $strSubPath ($oStorageRepo->list(\n"
"STORAGE_REPO_ARCHIVE . \"/${strArchiveId}/${strPath}\", {strExpression => \"^[0-F]{24}.*\\$\"}))\n"
"{\n"
"$bRemove = true;\n"
"\n\n"
"foreach my $oArchiveRange (@oyArchiveRange)\n"
"{\n"
"if (substr($strSubPath, 0, 24) ge $$oArchiveRange{start} &&\n"
"(!defined($$oArchiveRange{stop}) || substr($strSubPath, 0, 24) le $$oArchiveRange{stop}))\n"
"{\n"
"$bRemove = false;\n"
"last;\n"
"}\n"
"}\n"
"\n\n"
"if ($bRemove)\n"
"{\n"
"$oStorageRepo->remove(STORAGE_REPO_ARCHIVE . \"/${strArchiveId}/${strSubPath}\");\n"
"\n"
"logDebugMisc($strOperation, \"remove WAL segment: ${strArchiveId}/${strSubPath}\");\n"
"\n\n"
"$self->logExpire($strArchiveId, substr($strSubPath, 0, 24));\n"
"}\n"
"else\n"
"{\n"
"\n"
"$self->logExpire($strArchiveId);\n"
"}\n"
"}\n"
"}\n"
"}\n"
"\n\n"
"if ($self->{iArchiveExpireTotal} == 0)\n"
"{\n"
"&log(DETAIL, \"no archive to remove, archiveId = ${strArchiveId}\");\n"
"}\n"
"}\n"
"}\n"
"}\n"
"}\n"
"}\n"
"\n\n"
"return logDebugReturn($strOperation);\n"
"}\n"
"\n"
"1;\n"
},
{
.name = "pgBackRest/InfoCommon.pm",
.data =
@ -9685,11 +9284,13 @@ static const EmbeddedModule embeddedModule[] =
"\n"
"use File::Basename qw(dirname);\n"
"\n"
"use pgBackRest::Backup::Info;\n"
"use pgBackRest::Common::Exception;\n"
"use pgBackRest::Common::Lock;\n"
"use pgBackRest::Common::Log;\n"
"use pgBackRest::Config::Config;\n"
"use pgBackRest::Protocol::Helper;\n"
"use pgBackRest::Protocol::Storage::Helper;\n"
"use pgBackRest::Storage::Helper;\n"
"use pgBackRest::Version;\n"
"\n\n\n\n"
@ -9879,11 +9480,7 @@ static const EmbeddedModule embeddedModule[] =
"\n\n\n"
"elsif (cfgCommandTest(CFGCMD_EXPIRE))\n"
"{\n"
"\n"
"require pgBackRest::Expire;\n"
"pgBackRest::Expire->import();\n"
"\n"
"new pgBackRest::Expire()->process();\n"
"new pgBackRest::Backup::Info(storageRepo()->pathGet(STORAGE_REPO_BACKUP));\n"
"}\n"
"}\n"
"}\n"

7
test/define.yaml

@ -691,6 +691,13 @@ unit:
coverage:
command/control/control: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: expire
total: 6
coverage:
command/expire/expire: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: help
total: 4

2
test/expect/mock-all-002.log

@ -3353,6 +3353,7 @@ P00 INFO: new backup label = [BACKUP-DIFF-6]
P00 INFO: backup command end: completed successfully
P00 INFO: expire command begin
P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired
P00 INFO: http statistics:[HTTP-STATISTICS]
P00 INFO: expire command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
@ -3560,6 +3561,7 @@ P00 INFO: new backup label = [BACKUP-DIFF-7]
P00 INFO: backup command end: completed successfully
P00 INFO: expire command begin
P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired
P00 INFO: http statistics:[HTTP-STATISTICS]
P00 INFO: expire command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf

2
test/expect/mock-expire-002.log

@ -476,6 +476,7 @@ P00 INFO: remove expired backup [BACKUP-FULL-1]
P00 DETAIL: archive retention on backup [BACKUP-FULL-2], archiveId = 9.2-1, start = 00000001000000000000000C
P00 DETAIL: remove archive: archiveId = 9.2-1, start = 000000010000000000000000, stop = 00000001000000000000000B
P00 DETAIL: archive retention on backup [BACKUP-FULL-3], archiveId = 9.3-2, start = 000000010000000000000000
P00 DETAIL: no archive to remove, archiveId = 9.3-2
P00 INFO: expire command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info
@ -1147,6 +1148,7 @@ P00 INFO: remove archive path: [TEST_PATH]/db-master/repo/archive/db/9.2-1
P00 DETAIL: archive retention on backup [BACKUP-FULL-4], archiveId = 9.3-2, start = 0000000100000000000000FF
P00 DETAIL: remove archive: archiveId = 9.3-2, start = 000000010000000000000000, stop = 0000000100000000000000FE
P00 DETAIL: archive retention on backup [BACKUP-FULL-5], archiveId = 10-3, start = 000000010000000000000000
P00 DETAIL: no archive to remove, archiveId = 10-3
P00 INFO: expire command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info

4
test/expect/mock-stanza-001.log

@ -535,8 +535,7 @@ P00 INFO: full backup size = 48MB
P00 INFO: new backup label = [BACKUP-FULL-1]
P00 INFO: backup command end: completed successfully
P00 INFO: expire command begin
P00 INFO: remove archive path: [TEST_PATH]/db-master/repo/archive/db/9.3-1
P00 INFO: full backup total < 2 - using oldest full backup for 9.4-2 archive retention
P00 INFO: full backup total < 2 - using oldest full backup for 9.3-1 archive retention
P00 INFO: expire command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
@ -691,7 +690,6 @@ P00 INFO: full backup size = 48MB
P00 INFO: new backup label = [BACKUP-FULL-2]
P00 INFO: backup command end: completed successfully
P00 INFO: expire command begin
P00 INFO: remove archive path: [TEST_PATH]/db-master/repo/archive/db/10.0-3
P00 INFO: expire command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf

5
test/expect/mock-stanza-002.log

@ -313,6 +313,8 @@ P00 INFO: backup command end: completed successfully
P00 INFO: expire command begin
P00 INFO: remove archive path: /archive/db/9.3-1
P00 INFO: full backup total < 2 - using oldest full backup for 9.4-2 archive retention
P00 DETAIL: tls statistics:[TLS-STATISTICS]
P00 INFO: http statistics:[HTTP-STATISTICS]
P00 INFO: expire command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
@ -496,7 +498,8 @@ P00 INFO: full backup size = 48MB
P00 INFO: new backup label = [BACKUP-FULL-2]
P00 INFO: backup command end: completed successfully
P00 INFO: expire command begin
P00 INFO: remove archive path: /archive/db/10.0-3
P00 DETAIL: tls statistics:[TLS-STATISTICS]
P00 INFO: http statistics:[HTTP-STATISTICS]
P00 INFO: expire command end: completed successfully
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf

12
test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm

@ -22,7 +22,6 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Expire;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
@ -261,7 +260,6 @@ sub run
$self->configTestLoad(CFGCMD_EXPIRE);
$strDescription = 'Expiration cannot occur due to info file db mismatch';
my $oExpire = new pgBackRest::Expire();
# Mismatched version
$oHostBackup->infoMunge(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
@ -273,11 +271,6 @@ sub run
{&INFO_ARCHIVE_KEY_DB_VERSION => PG_VERSION_93,
&INFO_ARCHIVE_KEY_DB_ID => $self->dbSysId(PG_VERSION_95)}}});
$self->testException(sub {$oExpire->process()},
ERROR_FILE_INVALID,
"archive and backup database versions do not match\n" .
"HINT: has a stanza-upgrade been performed?");
# Restore the info file
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
@ -289,11 +282,6 @@ sub run
{'3' =>
{&INFO_ARCHIVE_KEY_DB_VERSION => PG_VERSION_95, &INFO_ARCHIVE_KEY_DB_ID => 6999999999999999999}}});
$self->testException(sub {$oExpire->process()},
ERROR_FILE_INVALID,
"archive and backup database versions do not match\n" .
"HINT: has a stanza-upgrade been performed?");
# Restore the info file
$oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE));
}

859
test/src/module/command/expireTest.c Normal file

@ -0,0 +1,859 @@
/***********************************************************************************************************************************
Test Command Control
***********************************************************************************************************************************/
#include "storage/posix/storage.h"
#include "common/harnessConfig.h"
#include "common/harnessInfo.h"
/***********************************************************************************************************************************
Helper functions
***********************************************************************************************************************************/
void
archiveGenerate(
Storage *storageTest, String *archiveStanzaPath, const unsigned int start, unsigned int end, const char *archiveId,
const char *majorWal)
{
// For simplicity, only allow 2 digits
if (end > 99)
end = 99;
String *wal = NULL;
for (unsigned int i = start; i <= end; i++)
{
if (i < 10)
wal = strNewFmt("%s0000000%u-9baedd24b61aa15305732ac678c4e2c102435a09", majorWal, i);
else
wal = strNewFmt("%s000000%u-9baedd24b61aa15305732ac678c4e2c102435a09", majorWal, i);
storagePutNP(
storageNewWriteNP(storageTest, strNewFmt("%s/%s/%s/%s", strPtr(archiveStanzaPath), archiveId, majorWal, strPtr(wal))),
BUFSTRDEF(BOGUS_STR));
}
}
String *
archiveExpectList(const unsigned int start, unsigned int end, const char *majorWal)
{
String *result = strNew("");
// For simplicity, only allow 2 digits
if (end > 99)
end = 99;
String *wal = NULL;
for (unsigned int i = start; i <= end; i++)
{
if (i < 10)
wal = strNewFmt("%s0000000%u-9baedd24b61aa15305732ac678c4e2c102435a09", majorWal, i);
else
wal = strNewFmt("%s000000%u-9baedd24b61aa15305732ac678c4e2c102435a09", majorWal, i);
if (strSize(result) == 0)
strCat(result, strPtr(wal));
else
strCatFmt(result, ", %s", strPtr(wal));
}
return result;
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
Storage *storageTest = storagePosixNew(
strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL);
String *backupStanzaPath = strNew("repo/backup/db");
String *backupInfoFileName = strNewFmt("%s/backup.info", strPtr(backupStanzaPath));
String *archiveStanzaPath = strNew("repo/archive/db");
String *archiveInfoFileName = strNewFmt("%s/archive.info", strPtr(archiveStanzaPath));
StringList *argListBase = strLstNew();
strLstAddZ(argListBase, "pgbackrest");
strLstAddZ(argListBase, "--stanza=db");
strLstAdd(argListBase, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAddZ(argListBase, "expire");
StringList *argListAvoidWarn = strLstDup(argListBase);
strLstAddZ(argListAvoidWarn, "--repo1-retention-full=1"); // avoid warning
String *backupInfoBase = strNew(
"[backup:current]\n"
"20181119-152138F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000004\",\"backup-archive-stop\":\"000000010000000000000004\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F_20181119-152152D={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000005\","
"\"backup-archive-stop\":\"000000010000000000000005\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F_20181119-152155I={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000006\","
"\"backup-archive-stop\":\"000000010000000000000006\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152800F_20181119-152152D\","
"\"backup-reference\":[\"20181119-152800F\",\"20181119-152800F_20181119-152152D\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"incr\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152900F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000007\",\"backup-archive-stop\":\"000000010000000000000007\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152900F_20181119-152600D={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000008\","
"\"backup-archive-stop\":\"000000010000000000000008\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152138F\",\"backup-reference\":[\"20181119-152900F\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"\n"
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6625592122879095702\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
"\"db-version\":\"9.4\"}");
// *****************************************************************************************************************************
if (testBegin("expireBackup()"))
{
// Create backup.info
storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName), harnessInfoChecksum(backupInfoBase));
InfoBackup *infoBackup = NULL;
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
// Create backup directories and manifest files
String *full1 = strNew("20181119-152138F");
String *full2 = strNew("20181119-152800F");
String *full1Path = strNewFmt("%s/%s", strPtr(backupStanzaPath), strPtr(full1));
String *full2Path = strNewFmt("%s/%s", strPtr(backupStanzaPath), strPtr(full2));
// Load Parameters
StringList *argList = strLstDup(argListAvoidWarn);
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full1Path), INFO_MANIFEST_FILE)),
BUFSTRDEF(BOGUS_STR)), "full1 put manifest");
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full1Path), INFO_MANIFEST_FILE ".copy")),
BUFSTRDEF(BOGUS_STR)), "full1 put manifest copy");
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full1Path), "bogus")),
BUFSTRDEF(BOGUS_STR)), "full1 put extra file");
TEST_RESULT_VOID(storagePathCreateNP(storageTest, full2Path), "full2 empty");
String *backupExpired = strNew("");
TEST_RESULT_VOID(expireBackup(infoBackup, full1, backupExpired), "expire backup with both manifest files");
TEST_RESULT_BOOL(
(strLstSize(storageListNP(storageTest, full1Path)) && strLstExistsZ(storageListNP(storageTest, full1Path), "bogus")),
true, " full1 - only manifest files removed");
TEST_RESULT_VOID(expireBackup(infoBackup, full2, backupExpired), "expire backup with no manifest - does not error");
TEST_RESULT_STR(
strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")),
"20181119-152800F_20181119-152152D, 20181119-152800F_20181119-152155I, 20181119-152900F, "
"20181119-152900F_20181119-152600D",
"only backups passed to expireBackup are removed from backup:current");
}
// *****************************************************************************************************************************
if (testBegin("expireFullBackup()"))
{
// Create backup.info
storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName), harnessInfoChecksum(backupInfoBase));
InfoBackup *infoBackup = NULL;
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
// Load Parameters
StringList *argList = strLstDup(argListBase);
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_UINT(expireFullBackup(infoBackup), 0, "retention-full not set");
harnessLogResult(
"P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n"
" HINT: to retain full backups indefinitely (without warning), "
"set option 'repo1-retention-full' to the maximum.");
//--------------------------------------------------------------------------------------------------------------------------
strLstAddZ(argList, "--repo1-retention-full=2");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_UINT(expireFullBackup(infoBackup), 1, "retention-full=2 - one full backup expired");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 5, " current backups reduced by 1 full - no dependencies");
TEST_RESULT_STR(
strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")),
"20181119-152800F, 20181119-152800F_20181119-152152D, 20181119-152800F_20181119-152155I, "
"20181119-152900F, 20181119-152900F_20181119-152600D", " remaining backups correct");
harnessLogResult("P00 INFO: expire full backup 20181119-152138F");
//--------------------------------------------------------------------------------------------------------------------------
argList = strLstDup(argListBase);
strLstAddZ(argList, "--repo1-retention-full=1");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_UINT(expireFullBackup(infoBackup), 3, "retention-full=1 - one full backup and dependencies expired");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, " current backups reduced by 1 full and dependencies");
TEST_RESULT_STR(
strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")),
"20181119-152900F, 20181119-152900F_20181119-152600D", " remaining backups correct");
harnessLogResult(
"P00 INFO: expire full backup set: 20181119-152800F, 20181119-152800F_20181119-152152D, "
"20181119-152800F_20181119-152155I");
//--------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_UINT(expireFullBackup(infoBackup), 0, "retention-full=1 - not enough backups to expire any");
TEST_RESULT_STR(
strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")),
"20181119-152900F, 20181119-152900F_20181119-152600D", " remaining backups correct");
}
// *****************************************************************************************************************************
if (testBegin("expireDiffBackup()"))
{
// Create backup.info
storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName), harnessInfoChecksum(backupInfoBase));
InfoBackup *infoBackup = NULL;
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
// Load Parameters
StringList *argList = strLstDup(argListAvoidWarn);
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_UINT(expireDiffBackup(infoBackup), 0, "retention-diff not set - nothing expired");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 6, " current backups not expired");
//--------------------------------------------------------------------------------------------------------------------------
// Add retention-diff
StringList *argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--repo1-retention-diff=6");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
TEST_RESULT_UINT(expireDiffBackup(infoBackup), 0, "retention-diff set - too soon to expire");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 6, " current backups not expired");
//--------------------------------------------------------------------------------------------------------------------------
strLstAddZ(argList, "--repo1-retention-diff=2");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_UINT(expireDiffBackup(infoBackup), 2, "retention-diff set - full considered in diff");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 4, " current backups reduced by 1 diff and dependent increment");
TEST_RESULT_STR(
strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")),
"20181119-152138F, 20181119-152800F, 20181119-152900F, 20181119-152900F_20181119-152600D",
" remaining backups correct");
harnessLogResult(
"P00 INFO: expire diff backup set: 20181119-152800F_20181119-152152D, 20181119-152800F_20181119-152155I");
TEST_RESULT_UINT(expireDiffBackup(infoBackup), 0, "retention-diff=2 but no more to expire");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 4, " current backups not reduced");
//--------------------------------------------------------------------------------------------------------------------------
// Create backup.info with two diff - oldest to be expired - no "set:"
storagePutNP(
storageNewWriteNP(storageTest, backupInfoFileName),
harnessInfoChecksumZ(
"[backup:current]\n"
"20181119-152800F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F_20181119-152152D={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000003\","
"\"backup-archive-stop\":\"000000010000000000000003\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F_20181119-152155D={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000004\","
"\"backup-archive-stop\":\"000000010000000000000004\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"\n"
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6625592122879095702\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
"\"db-version\":\"9.4\"}"));
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
// Load parameters
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-diff=1");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_UINT(expireDiffBackup(infoBackup), 1, "retention-diff set - only oldest diff expired");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, " current backups reduced by one");
TEST_RESULT_STR(
strPtr(strLstJoin(infoBackupDataLabelList(infoBackup, NULL), ", ")),
"20181119-152800F, 20181119-152800F_20181119-152155D",
" remaining backups correct");
harnessLogResult(
"P00 INFO: expire diff backup 20181119-152800F_20181119-152152D");
}
// *****************************************************************************************************************************
if (testBegin("removeExpiredBackup()"))
{
// Create backup.info
storagePutNP(
storageNewWriteNP(storageTest, backupInfoFileName),
harnessInfoChecksumZ(
"[backup:current]\n"
"20181119-152138F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"\n"
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6625592122879095702\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
"\"db-version\":\"9.4\"}"));
InfoBackup *infoBackup = NULL;
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
// Create backup directories, manifest files and other path/file
String *full = strNewFmt("%s/%s", strPtr(backupStanzaPath), "20181119-152100F");
String *diff = strNewFmt("%s/%s", strPtr(backupStanzaPath), "20181119-152100F_20181119-152152D");
String *otherPath = strNewFmt("%s/%s", strPtr(backupStanzaPath), "bogus");
String *otherFile = strNewFmt("%s/%s", strPtr(backupStanzaPath), "20181118-152100F_20181119-152152D.save");
String *full1 = strNewFmt("%s/%s", strPtr(backupStanzaPath), "20181119-152138F");
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full), "bogus")),
BUFSTRDEF(BOGUS_STR)), "put file");
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/%s", strPtr(full1), "somefile")),
BUFSTRDEF(BOGUS_STR)), "put file");
TEST_RESULT_VOID(storagePathCreateNP(storageTest, diff), "empty backup directory must not error on delete");
TEST_RESULT_VOID(storagePathCreateNP(storageTest, otherPath), "other path must not be removed");
TEST_RESULT_VOID(
storagePutNP(storageNewWriteNP(storageTest, otherFile),
BUFSTRDEF(BOGUS_STR)), "directory look-alike file must not be removed");
// Load Parameters
StringList *argList = strLstDup(argListBase);
strLstAddZ(argList, "--repo1-retention-full=1");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(removeExpiredBackup(infoBackup), "remove backups not in backup.info current");
harnessLogResult(
"P00 INFO: remove expired backup 20181119-152100F_20181119-152152D\n"
"P00 INFO: remove expired backup 20181119-152100F");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(storageTest, backupStanzaPath), sortOrderAsc), ", ")),
"20181118-152100F_20181119-152152D.save, 20181119-152138F, backup.info, bogus", " remaining file/directories correct");
//--------------------------------------------------------------------------------------------------------------------------
// Create backup.info without current backups
storagePutNP(
storageNewWriteNP(storageTest, backupInfoFileName),
harnessInfoChecksumZ(
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6625592122879095702\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
"\"db-version\":\"9.4\"}"));
TEST_ASSIGN(infoBackup,infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
TEST_RESULT_VOID(removeExpiredBackup(infoBackup), "remove backups - backup.info current empty");
harnessLogResult("P00 INFO: remove expired backup 20181119-152138F");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(storageTest, backupStanzaPath), sortOrderAsc), ", ")),
"20181118-152100F_20181119-152152D.save, backup.info, bogus", " remaining file/directories correct");
}
// *****************************************************************************************************************************
if (testBegin("removeExpiredArchive() & cmdExpire()"))
{
// Load Parameters
StringList *argList = strLstDup(argListBase);
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// Create backup.info without current backups
storagePutNP(
storageNewWriteNP(storageTest, backupInfoFileName),
harnessInfoChecksumZ(
"[db]\n"
"db-catalog-version=201409291\n"
"db-control-version=942\n"
"db-id=1\n"
"db-system-id=6625592122879095702\n"
"db-version=\"9.4\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
"\"db-version\":\"9.4\"}"));
InfoBackup *infoBackup = NULL;
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention not set");
harnessLogResult(
"P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n"
" HINT: to retain full backups indefinitely (without warning), "
"set option 'repo1-retention-full' to the maximum.\n"
"P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired");
//--------------------------------------------------------------------------------------------------------------------------
// Set archive retention, archive retention type default but no current backups - code path test
strLstAddZ(argList, "--repo1-retention-archive=4");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention set, retention type default, no current backups");
harnessLogResult(
"P00 WARN: option repo1-retention-full is not set, the repository may run out of space\n"
" HINT: to retain full backups indefinitely (without warning), "
"set option 'repo1-retention-full' to the maximum.");
//--------------------------------------------------------------------------------------------------------------------------
// Create backup.info with current backups spread over different timelines
storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName),
harnessInfoChecksumZ(
"[backup:current]\n"
"20181119-152138F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000020000000000000002\",\"backup-archive-stop\":\"000000020000000000000002\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F_20181119-152152D={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000020000000000000004\","
"\"backup-archive-stop\":\"000000020000000000000005\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F_20181119-152155I={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000020000000000000007\","
"\"backup-archive-stop\":\"000000020000000000000007\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152800F_20181119-152152D\","
"\"backup-reference\":[\"20181119-152800F\",\"20181119-152800F_20181119-152152D\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"incr\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F_20181119-152252D={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000020000000000000009\","
"\"backup-archive-stop\":\"000000020000000000000009\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152800F\",\"backup-reference\":[\"20181119-152800F\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152900F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000003\",\"backup-archive-stop\":\"000000010000000000000004\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152900F_20181119-152500I={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\",\"backup-archive-start\":\"000000010000000000000006\","
"\"backup-archive-stop\":\"000000010000000000000006\",\"backup-info-repo-size\":2369186,"
"\"backup-info-repo-size-delta\":346,\"backup-info-size\":20162900,\"backup-info-size-delta\":8428,"
"\"backup-prior\":\"20181119-152900F\",\"backup-reference\":[\"20181119-152900F\"],"
"\"backup-timestamp-start\":1542640912,\"backup-timestamp-stop\":1542640915,\"backup-type\":\"diff\","
"\"db-id\":2,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"\n"
"[db]\n"
"db-catalog-version=201707211\n"
"db-control-version=1002\n"
"db-id=2\n"
"db-system-id=6626363367545678089\n"
"db-version=\"10\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
"\"db-version\":\"9.4\"}\n"
"2={\"db-catalog-version\":201707211,\"db-control-version\":1002,\"db-system-id\":6626363367545678089,"
"\"db-version\":\"10\"}\n"));
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
storagePutNP(
storageNewWriteNP(storageTest, archiveInfoFileName),
harnessInfoChecksumZ(
"[db]\n"
"db-id=2\n"
"db-system-id=6626363367545678089\n"
"db-version=\"10\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6625592122879095702,\"db-version\":\"9.4\"}\n"
"2={\"db-id\":6626363367545678089,\"db-version\":\"10\"}"));
TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "no archive on disk");
//--------------------------------------------------------------------------------------------------------------------------
archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "9.4-1", "0000000100000000");
archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "9.4-1", "0000000200000000");
archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "10-2", "0000000100000000");
TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention type = full (default), repo1-retention-archive=4");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(2, 10, "0000000100000000")),
" only 9.4-1/0000000100000000/000000010000000000000001 removed");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(1, 10, "0000000200000000")),
" none removed from 9.4-1/0000000200000000 - crossing timelines to play through PITR");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(3, 10, "0000000100000000")),
" 000000010000000000000001 and 000000010000000000000002 removed from 10-2/0000000100000000");
harnessLogResult(
"P00 INFO: full backup total < 4 - using oldest full backup for 9.4-1 archive retention\n"
"P00 INFO: full backup total < 4 - using oldest full backup for 10-2 archive retention");
//--------------------------------------------------------------------------------------------------------------------------
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=2");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention type = full (default), repo1-retention-archive=2");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(2, 2, "0000000100000000")),
" only 9.4-1/0000000100000000/000000010000000000000002 remains in major wal 1");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(2, 10, "0000000200000000")),
" only 9.4-1/0000000200000000/000000010000000000000001 removed from major wal 2");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(3, 10, "0000000100000000")),
" none removed from 10-2/0000000100000000");
//--------------------------------------------------------------------------------------------------------------------------
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=1");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "archive retention type = full (default), repo1-retention-archive=1");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(2, 2, "0000000100000000")),
" only 9.4-1/0000000100000000/000000010000000000000002 remains in major wal 1");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(2, 10, "0000000200000000")),
" nothing removed from 9.4-1/0000000200000000 major wal 2 - each archiveId must have one backup to play through PITR");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(3, 10, "0000000100000000")),
" none removed from 10-2/0000000100000000");
//--------------------------------------------------------------------------------------------------------------------------
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=2");
strLstAddZ(argList, "--repo1-retention-archive-type=diff");
strLstAddZ(argList, "--repo1-retention-diff=2");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(
removeExpiredArchive(infoBackup),
"full counts as differential and incremental associated with differential expires");
String *result = strNew("");
strCatFmt(
result,
"%s, %s, %s, %s",
strPtr(archiveExpectList(2, 2, "0000000200000000")),
strPtr(archiveExpectList(4, 5, "0000000200000000")),
strPtr(archiveExpectList(7, 7, "0000000200000000")),
strPtr(archiveExpectList(9, 10, "0000000200000000")));
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(2, 2, "0000000100000000")),
" only 9.4-1/0000000100000000/000000010000000000000002 remains in major wal 1");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")),
strPtr(result),
" all in-between removed from 9.4-1/0000000200000000 major wal 2 - last backup able to play through PITR");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(3, 10, "0000000100000000")),
" none removed from 10-2/0000000100000000");
//--------------------------------------------------------------------------------------------------------------------------
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=4");
strLstAddZ(argList, "--repo1-retention-archive-type=incr");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// Regenerate archive
archiveGenerate(storageTest, archiveStanzaPath, 1, 10, "9.4-1", "0000000200000000");
TEST_RESULT_VOID(removeExpiredArchive(infoBackup), "differential and full count as an incremental");
result = strNew("");
strCatFmt(
result,
"%s, %s, %s",
strPtr(archiveExpectList(2, 2, "0000000200000000")),
strPtr(archiveExpectList(4, 5, "0000000200000000")),
strPtr(archiveExpectList(7, 10, "0000000200000000")));
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(2, 2, "0000000100000000")),
" only 9.4-1/0000000100000000/000000010000000000000002 remains in major wal 1");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000200000000")), sortOrderAsc), ", ")),
strPtr(result),
" incremental and after remain in 9.4-1/0000000200000000 major wal 2");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "10-2", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(3, 10, "0000000100000000")),
" none removed from 10-2/0000000100000000");
//--------------------------------------------------------------------------------------------------------------------------
argList = strLstDup(argListBase);
strLstAddZ(argList, "--repo1-retention-full=2");
strLstAddZ(argList, "--repo1-retention-diff=3");
strLstAddZ(argList, "--repo1-retention-archive=2");
strLstAddZ(argList, "--repo1-retention-archive-type=diff");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(cmdExpire(), "expire last backup in archive sub path and remove sub path");
TEST_RESULT_BOOL(
storagePathExistsNP(storageTest, strNewFmt("%s/%s", strPtr(archiveStanzaPath), "9.4-1/0000000100000000")),
false, " archive sub path removed");
harnessLogResult("P00 INFO: expire full backup 20181119-152138F");
//--------------------------------------------------------------------------------------------------------------------------
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=1");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(cmdExpire(), "expire last backup in archive path and remove path");
TEST_RESULT_BOOL(
storagePathExistsNP(storageTest, strNewFmt("%s/%s", strPtr(archiveStanzaPath), "9.4-1")),
false, " archive path removed");
harnessLogResult(strPtr(strNewFmt(
"P00 INFO: expire full backup set: 20181119-152800F, 20181119-152800F_20181119-152152D, "
"20181119-152800F_20181119-152155I, 20181119-152800F_20181119-152252D\n"
"P00 INFO: remove archive path: %s/%s/9.4-1", testPath(), strPtr(archiveStanzaPath))));
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), " get backup.info");
TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, " backup.info updated on disk");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(infoBackupDataLabelList(infoBackup, NULL), sortOrderAsc), ", ")),
"20181119-152900F, 20181119-152900F_20181119-152500I", " remaining current backups correct");
//--------------------------------------------------------------------------------------------------------------------------
storagePutNP(storageNewWriteNP(storageTest, backupInfoFileName),
harnessInfoChecksumZ(
"[backup:current]\n"
"20181119-152138F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000002\",\"backup-archive-stop\":\"000000010000000000000002\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152800F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"20181119-152900F={"
"\"backrest-format\":5,\"backrest-version\":\"2.08dev\","
"\"backup-archive-start\":\"000000010000000000000004\",\"backup-archive-stop\":\"000000010000000000000004\","
"\"backup-info-repo-size\":2369186,\"backup-info-repo-size-delta\":2369186,"
"\"backup-info-size\":20162900,\"backup-info-size-delta\":20162900,"
"\"backup-timestamp-start\":1542640898,\"backup-timestamp-stop\":1542640911,\"backup-type\":\"full\","
"\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false,\"option-backup-standby\":false,"
"\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n"
"\n"
"[db]\n"
"db-catalog-version=201707211\n"
"db-control-version=1002\n"
"db-id=2\n"
"db-system-id=6626363367545678089\n"
"db-version=\"10\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702,"
"\"db-version\":\"9.4\"}\n"
"2={\"db-catalog-version\":201707211,\"db-control-version\":1002,\"db-system-id\":6626363367545678089,"
"\"db-version\":\"10\"}\n"));
TEST_ASSIGN(infoBackup, infoBackupNewLoad(storageTest, backupInfoFileName, cipherTypeNone, NULL), "get backup.info");
archiveGenerate(storageTest, archiveStanzaPath, 1, 5, "9.4-1", "0000000100000000");
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=2");
strLstAddZ(argList, "--repo1-retention-archive-type=full");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(
removeExpiredArchive(infoBackup), "backup selected for retention does not have archive-start so do nothing");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(1, 5, "0000000100000000")),
" nothing removed from 9.4-1/0000000100000000");
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=4");
strLstAddZ(argList, "--repo1-retention-archive-type=incr");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_VOID(
removeExpiredArchive(infoBackup), "full count as incr but not enough backups, retention set to first full");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(
storageTest, strNewFmt("%s/%s/%s", strPtr(archiveStanzaPath), "9.4-1", "0000000100000000")), sortOrderAsc), ", ")),
strPtr(archiveExpectList(2, 5, "0000000100000000")),
" only removed archive prior to first full");
harnessLogResult(
"P00 INFO: full backup total < 4 - using oldest full backup for 9.4-1 archive retention");
argList = strLstDup(argListAvoidWarn);
strLstAddZ(argList, "--repo1-retention-archive=1");
strLstAddZ(argList, "--repo1-retention-archive-type=full");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
harnessLogLevelSet(logLevelDetail);
TEST_RESULT_VOID(
removeExpiredArchive(infoBackup), "backup earlier than selected for retention does not have archive-start");
harnessLogResult(
"P00 DETAIL: archive retention on backup 20181119-152138F, archiveId = 9.4-1, start = 000000010000000000000002,"
" stop = 000000010000000000000002\n"
"P00 DETAIL: archive retention on backup 20181119-152900F, archiveId = 9.4-1, start = 000000010000000000000004\n"
"P00 DETAIL: remove archive: archiveId = 9.4-1, start = 000000010000000000000003, stop = 000000010000000000000003");
harnessLogLevelReset();
}
// *****************************************************************************************************************************
if (testBegin("sortArchiveId()"))
{
StringList *list = strLstNew();
strLstAddZ(list, "10-4");
strLstAddZ(list, "11-10");
strLstAddZ(list, "9.6-1");
TEST_RESULT_STR(strPtr(strLstJoin(sortArchiveId(list, sortOrderAsc), ", ")), "9.6-1, 10-4, 11-10", "sort ascending");
strLstAddZ(list, "9.4-2");
TEST_RESULT_STR(
strPtr(strLstJoin(sortArchiveId(list, sortOrderDesc), ", ")), "11-10, 10-4, 9.4-2, 9.6-1", "sort descending");
}
FUNCTION_HARNESS_RESULT_VOID();
}