db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-06-20 01:17:49 +02:00
Code Issues Releases Activity

Add time-based retention for full backups.

Browse Source 
The --repo-retention-full-type option allows retention of full backups based on a time period, specified in days.

The new option will default to 'count' and therefore will not affect current installations. Setting repo-retention-full-type to 'time' will allow the user to use a time period, in days, to indicate full backup retention. Using this method, a full backup can be expired only if the time the backup completed is older than the number of days set with repo-retention-full (calculated from the moment the 'expire' command is run) and at least one full backup meets the retention period. If archive retention has not been configured, then the default settings will expire archives that are prior to the oldest retained full backup. For example, if there are three full backups ending in times that are 25 days old (F1), 20 days old (F2) and 10 days old (F3), then if the full retention period is 15 days, then only F1 will be expired; F2 will be retained because F1 is not at least 15 days old.
This commit is contained in:
Cynthia Shang
2020-05-08 15:25:03 -04:00
committed by GitHub
parent e873ad6da0
commit cdebfb09e0
21 changed files with 572 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/command/expire/expire.c
+99 -6
View File
@@ -6,6 +6,7 @@ Expire Command
#include "command/archive/common.h"
#include "command/backup/common.h"
#include "command/control/common.h"
#include "common/time.h"
#include "common/type/list.h"
#include "common/debug.h"
#include "common/regExp.h"
@@ -17,7 +18,6 @@ Expire Command
#include "storage/helper.h"
#include <stdlib.h>
#include <stdio.h>
/***********************************************************************************************************************************
Helper functions and structures
@@ -286,6 +286,80 @@ expireFullBackup(InfoBackup *infoBackup)
FUNCTION_LOG_RETURN(UINT, result);
}
/***********************************************************************************************************************************
Expire backups based on time
***********************************************************************************************************************************/
static unsigned int
expireTimeBasedBackup(InfoBackup *infoBackup, const time_t minTimestamp)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_PARAM(TIME, minTimestamp);
FUNCTION_LOG_END();
ASSERT(infoBackup != NULL);
ASSERT(minTimestamp > 0);
unsigned int result = 0;
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the list of full backups
StringList *currentBackupList = strLstSort(infoBackupDataLabelList(infoBackup, backupRegExpP(.full = true)), sortOrderAsc);
unsigned int backupIdx = strLstSize(currentBackupList);
// Find out the point where we will have to stop purging backups. Starting with the newest backup (the end of the list),
// find the first backup that is older than the expire time period by checking the backup stop time. This way, if the
// backups are F1 D1a D1b D1c F2 D2a D2b F3 D3a D3b and the expiration time period is at D2b, then purge only F1, D1a, D1b
// and D1c, and keep the next full backups (F2 and F3) and all intermediate non-full backups.
if (backupIdx > 0)
{
const String *lastBackupLabelToKeep = NULL;
do
{
backupIdx--;
InfoBackupData *info = infoBackupDataByLabel(infoBackup, strLstGet(currentBackupList, backupIdx));
lastBackupLabelToKeep = info->backupLabel;
// We can start deleting before this backup. This way, we keep one full backup and its dependents.
if (info->backupTimestampStop < minTimestamp)
break;
}
while (backupIdx != 0);
// Count number of full backups being expired
unsigned int numFullExpired = 0;
// Since expireBackup will remove the requested entry from the backup list, we keep checking the first entry which is
// always the oldest so if it is not the backup to keep then we can remove it
while (!strEq(infoBackupData(infoBackup, 0).backupLabel, lastBackupLabelToKeep))
{
StringList *backupExpired = expireBackup(infoBackup, infoBackupData(infoBackup, 0).backupLabel);
result += strLstSize(backupExpired);
numFullExpired++;
// Log the expired backups. If there is more than one backup, then prepend "set:"
LOG_INFO_FMT(
"expire time-based backup %s%s", (strLstSize(backupExpired) > 1 ? "set: " : ""),
strPtr(strLstJoin(backupExpired, ", ")));
}
if (strEqZ(cfgOptionStr(cfgOptRepoRetentionArchiveType), CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL) &&
!cfgOptionTest(cfgOptRepoRetentionArchive) && numFullExpired > 0)
{
cfgOptionSet(
cfgOptRepoRetentionArchive, cfgSourceDefault, varNewUInt(strLstSize(currentBackupList) - numFullExpired));
}
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(UINT, result);
}
/***********************************************************************************************************************************
Log detailed information about archive logs removed
***********************************************************************************************************************************/
@@ -307,10 +381,11 @@ logExpire(ArchiveExpired *archiveExpire, String *archiveId)
Process archive retention
***********************************************************************************************************************************/
static void
removeExpiredArchive(InfoBackup *infoBackup)
removeExpiredArchive(InfoBackup *infoBackup, bool timeBasedFullRetention)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_PARAM(BOOL, timeBasedFullRetention);
FUNCTION_LOG_END();
ASSERT(infoBackup != NULL);
@@ -325,7 +400,13 @@ removeExpiredArchive(InfoBackup *infoBackup)
// cfgLoadUpdateOption based on certain rules.
if (archiveRetention == 0)
{
LOG_INFO_FMT("option '%s' is not set - archive logs will not be expired", cfgOptionName(cfgOptRepoRetentionArchive));
String *msg = strNew("- archive logs will not be expired");
// Only notify user if not time-based retention
if (!timeBasedFullRetention)
LOG_INFO_FMT("option '%s' is not set %s", cfgOptionName(cfgOptRepoRetentionArchive), strPtr(msg));
else
LOG_INFO_FMT("time-based archive retention not met %s", strPtr(msg));
}
else
{
@@ -764,16 +845,28 @@ cmdExpire(void)
cfgOptionStrNull(cfgOptRepoCipherPass));
const String *adhocBackupLabel = NULL;
bool timeBasedFullRetention = strEqZ(
cfgOptionStr(cfgOptRepoRetentionFullType), CFGOPTVAL_TMP_REPO_RETENTION_FULL_TYPE_TIME);
// If the --set option is valid (i.e. expire is called on its own) and is set then attempt to expire the requested backup
if (cfgOptionValid(cfgOptSet) && cfgOptionTest(cfgOptSet))
if (cfgOptionTest(cfgOptSet))
{
adhocBackupLabel = cfgOptionStr(cfgOptSet);
expireAdhocBackup(infoBackup, adhocBackupLabel);
}
else
{
expireFullBackup(infoBackup);
// If time-based retention for full backups is set, then expire based on time period
if (timeBasedFullRetention)
{
// If a time period was provided then run time-based expiration otherwise do nothing (the user has already been
// warned by the config system that retention-full was not set)
if (cfgOptionTest(cfgOptRepoRetentionFull))
expireTimeBasedBackup(infoBackup, time(NULL) - (time_t)(cfgOptionUInt(cfgOptRepoRetentionFull) * SEC_PER_DAY));
}
else
expireFullBackup(infoBackup);
expireDiffBackup(infoBackup);
}
@@ -787,7 +880,7 @@ cmdExpire(void)
// Remove all files on disk that are now expired
removeExpiredBackup(infoBackup, adhocBackupLabel);
removeExpiredArchive(infoBackup);
removeExpiredArchive(infoBackup, timeBasedFullRetention);
}
MEM_CONTEXT_TEMP_END();
src/common/time.h
+1
View File
@@ -16,6 +16,7 @@ typedef uint64_t TimeMSec;
Constants describing number of sub-units in an interval
***********************************************************************************************************************************/
#define MSEC_PER_SEC ((TimeMSec)1000)
#define SEC_PER_DAY ((time_t)86400)
/***********************************************************************************************************************************
Functions
src/config/config.auto.c
+9
View File
@@ -434,6 +434,7 @@ STRING_EXTERN(CFGOPT_REPO1_RETENTION_ARCHIVE_STR, CFGOPT_REPO1
STRING_EXTERN(CFGOPT_REPO1_RETENTION_ARCHIVE_TYPE_STR, CFGOPT_REPO1_RETENTION_ARCHIVE_TYPE);
STRING_EXTERN(CFGOPT_REPO1_RETENTION_DIFF_STR, CFGOPT_REPO1_RETENTION_DIFF);
STRING_EXTERN(CFGOPT_REPO1_RETENTION_FULL_STR, CFGOPT_REPO1_RETENTION_FULL);
STRING_EXTERN(CFGOPT_REPO1_RETENTION_FULL_TYPE_STR, CFGOPT_REPO1_RETENTION_FULL_TYPE);
STRING_EXTERN(CFGOPT_REPO1_S3_BUCKET_STR, CFGOPT_REPO1_S3_BUCKET);
STRING_EXTERN(CFGOPT_REPO1_S3_CA_FILE_STR, CFGOPT_REPO1_S3_CA_FILE);
STRING_EXTERN(CFGOPT_REPO1_S3_CA_PATH_STR, CFGOPT_REPO1_S3_CA_PATH);
@@ -1680,6 +1681,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoRetentionFull)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
CONFIG_OPTION_NAME(CFGOPT_REPO1_RETENTION_FULL_TYPE)
CONFIG_OPTION_INDEX(0)
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoRetentionFullType)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
src/config/config.auto.h
+4 -1
View File
@@ -355,6 +355,8 @@ Option constants
STRING_DECLARE(CFGOPT_REPO1_RETENTION_DIFF_STR);
#define CFGOPT_REPO1_RETENTION_FULL "repo1-retention-full"
STRING_DECLARE(CFGOPT_REPO1_RETENTION_FULL_STR);
#define CFGOPT_REPO1_RETENTION_FULL_TYPE "repo1-retention-full-type"
STRING_DECLARE(CFGOPT_REPO1_RETENTION_FULL_TYPE_STR);
#define CFGOPT_REPO1_S3_BUCKET "repo1-s3-bucket"
STRING_DECLARE(CFGOPT_REPO1_S3_BUCKET_STR);
#define CFGOPT_REPO1_S3_CA_FILE "repo1-s3-ca-file"
@@ -420,7 +422,7 @@ Option constants
#define CFGOPT_TYPE "type"
STRING_DECLARE(CFGOPT_TYPE_STR);
#define CFG_OPTION_TOTAL 183
#define CFG_OPTION_TOTAL 184
/***********************************************************************************************************************************
Command enum
@@ -605,6 +607,7 @@ typedef enum
cfgOptRepoRetentionArchiveType,
cfgOptRepoRetentionDiff,
cfgOptRepoRetentionFull,
cfgOptRepoRetentionFullType,
cfgOptRepoS3Bucket,
cfgOptRepoS3CaFile,
cfgOptRepoS3CaPath,
src/config/config.h
+3
View File
@@ -50,6 +50,9 @@ Constants for configuration options.
#define CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL "full"
#define CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_INCR "incr"
#define CFGOPTVAL_TMP_REPO_RETENTION_FULL_TYPE_COUNT "count"
#define CFGOPTVAL_TMP_REPO_RETENTION_FULL_TYPE_TIME "time"
/***********************************************************************************************************************************
Command Functions
src/config/define.auto.c
+51 -4
View File
@@ -3404,9 +3404,11 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
"NOTE: WAL segments required to make a backup consistent are always retained until the backup is expired regardless of "
"how this option is configured.\n"
"\n"
"If this value is not set, then the archive to expire will default to the repo-retention-full (or repo-retention-diff) "
"value corresponding to the repo-retention-archive-type if set to full (or diff). This will ensure that WAL is "
"only expired for backups that are already expired.\n"
"If this value is not set and repo-retention-full-type is count (default), then the archive to expire will default to "
"the repo-retention-full (or repo-retention-diff) value corresponding to the repo-retention-archive-type if set to "
"full (or diff). This will ensure that WAL is only expired for backups that are already expired. If "
"repo-retention-full-type is time, then this value will default to removing archives that are earlier than the "
"oldest full backup retained after satisfying the repo-retention-full setting.\n"
"\n"
"This option must be set if repo-retention-archive-type is set to incr. If disk space is at a premium, then this "
"setting, in conjunction with repo-retention-archive-type, can be used to aggressively expire WAL segments. "
@@ -3521,7 +3523,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
CFGDEFDATA_OPTION_SECURE(false)
CFGDEFDATA_OPTION_HELP_SECTION("repository")
CFGDEFDATA_OPTION_HELP_SUMMARY("Number of full backups to retain.")
CFGDEFDATA_OPTION_HELP_SUMMARY("Full backup retention count/time.")
CFGDEFDATA_OPTION_HELP_DESCRIPTION
(
"When a full backup expires, all differential and incremental backups associated with the full backup will also "
@@ -3543,6 +3545,51 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
CFGDEFDATA_OPTION_NAME("repo-retention-full-type")
CFGDEFDATA_OPTION_REQUIRED(true)
CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString)
CFGDEFDATA_OPTION_INTERNAL(false)
CFGDEFDATA_OPTION_INDEX_TOTAL(1)
CFGDEFDATA_OPTION_SECURE(false)
CFGDEFDATA_OPTION_HELP_SECTION("repository")
CFGDEFDATA_OPTION_HELP_SUMMARY("Retention type for full backups.")
CFGDEFDATA_OPTION_HELP_DESCRIPTION
(
"Determines whether the repo-retention-full setting represents a time period (days) or count of full backups to keep. "
"If set to time then full backups older than repo-retention-full will be removed from the repository if there is "
"at least one backup that is equal to or greater than the repo-retention-full setting. For example, if "
"repo-retention-full is 30 (days) and there are 2 full backups: one 25 days old and one 35 days old, no full "
"backups will be expired because expiring the 35 day old backup would leave only the 25 day old backup, which "
"would violate the 30 day retention policy of having at least one backup 30 days old before an older one can be "
"expired. Archived WAL older than the oldest full backup remaining will be automatically expired unless "
"repo-retention-archive-type and repo-retention-archive are explicitly set."
)
CFGDEFDATA_OPTION_COMMAND_LIST
(
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire)
)
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
(
"count",
"time"
)
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("count")
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
src/config/define.auto.h
+1
View File
@@ -126,6 +126,7 @@ typedef enum
cfgDefOptRepoRetentionArchiveType,
cfgDefOptRepoRetentionDiff,
cfgDefOptRepoRetentionFull,
cfgDefOptRepoRetentionFullType,
cfgDefOptRepoS3Bucket,
cfgDefOptRepoS3CaFile,
cfgDefOptRepoS3CaPath,
src/config/load.c
+11 -8
View File
@@ -135,17 +135,18 @@ cfgLoadUpdateOption(void)
}
// Warn when repo-retention-full is not set on a configured repo
if (!cfgCommandHelp() && cfgOptionValid(cfgOptRepoRetentionFull) && cfgCommandRole() == cfgCmdRoleDefault)
if (!cfgCommandHelp() && cfgOptionValid(cfgOptRepoRetentionFullType) && cfgCommandRole() == cfgCmdRoleDefault)
{
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptRepoType); optionIdx++)
{
// If the repo-type is defined, then see if corresponding retention-full is set
if (cfgOptionTest(cfgOptRepoType + optionIdx) && !cfgOptionTest(cfgOptRepoRetentionFull + optionIdx))
if (cfgOptionTest(cfgOptRepoType + optionIdx) && !(cfgOptionTest(cfgOptRepoRetentionFull + optionIdx)))
{
LOG_WARN_FMT(
"option %s is not set, the repository may run out of space"
"\nHINT: to retain full backups indefinitely (without warning), set option '%s' to the maximum.",
cfgOptionName(cfgOptRepoRetentionFull + optionIdx),
"option '%s' is not set for '%s=%s', the repository may run out of space"
"\nHINT: to retain full backups indefinitely (without warning), set option '%s' to the maximum.",
cfgOptionName(cfgOptRepoRetentionFull + optionIdx), cfgOptionName(cfgOptRepoRetentionFullType + optionIdx),
strPtr(cfgOptionStr(cfgOptRepoRetentionFullType + optionIdx)),
cfgOptionName(cfgOptRepoRetentionFull + optionIdx));
}
}
@@ -166,11 +167,13 @@ cfgLoadUpdateOption(void)
// If the archive retention is not explicitly set then determine what it should be defaulted to
if (!cfgOptionTest(cfgOptRepoRetentionArchive + optionIdx))
{
// If repo-retention-archive-type is default, then if repo-retention-full is set, set the repo-retention-archive
// to this value, else ignore archiving
// If repo-retention-archive-type is default (full), then if repo-retention-full is set, set the
// repo-retention-archive to this value when retention-full-type is 'count', else ignore archiving. If
// retention-full-type is 'time' then the the expire command will default the archive retention accordingly.
if (strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL))
{
if (cfgOptionTest(cfgOptRepoRetentionFull + optionIdx))
if (strEqZ(cfgOptionStr(cfgOptRepoRetentionFullType + optionIdx), CFGOPTVAL_TMP_REPO_RETENTION_FULL_TYPE_COUNT)
&& cfgOptionTest(cfgOptRepoRetentionFull + optionIdx))
{
cfgOptionSet(cfgOptRepoRetentionArchive + optionIdx, cfgSourceDefault,
VARUINT(cfgOptionUInt(cfgOptRepoRetentionFull + optionIdx)));
src/config/parse.auto.c
+13
View File
@@ -2005,6 +2005,18 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_DEPRECATE_FLAG | cfgOptRepoRetentionFull,
},
// repo-retention-full-type option
// -----------------------------------------------------------------------------------------------------------------------------
{
.name = CFGOPT_REPO1_RETENTION_FULL_TYPE,
.has_arg = required_argument,
.val = PARSE_OPTION_FLAG | cfgOptRepoRetentionFullType,
},
{
.name = "reset-" CFGOPT_REPO1_RETENTION_FULL_TYPE,
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoRetentionFullType,
},
// repo-s3-bucket option and deprecations
// -----------------------------------------------------------------------------------------------------------------------------
{
@@ -2592,6 +2604,7 @@ static const ConfigOption optionResolveOrder[] =
cfgOptRepoRetentionArchiveType,
cfgOptRepoRetentionDiff,
cfgOptRepoRetentionFull,
cfgOptRepoRetentionFullType,
cfgOptRepoType,
cfgOptResume,
cfgOptSckBlock,